lazycogs 0.1.2__tar.gz → 0.2.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {lazycogs-0.1.2 → lazycogs-0.2.0}/PKG-INFO +4 -4
- {lazycogs-0.1.2 → lazycogs-0.2.0}/README.md +3 -3
- {lazycogs-0.1.2 → lazycogs-0.2.0}/pyproject.toml +1 -1
- {lazycogs-0.1.2 → lazycogs-0.2.0}/src/lazycogs/__init__.py +3 -0
- {lazycogs-0.1.2 → lazycogs-0.2.0}/src/lazycogs/_backend.py +61 -94
- {lazycogs-0.1.2 → lazycogs-0.2.0}/src/lazycogs/_chunk_reader.py +62 -3
- {lazycogs-0.1.2 → lazycogs-0.2.0}/src/lazycogs/_core.py +3 -3
- lazycogs-0.2.0/src/lazycogs/_executor.py +128 -0
- {lazycogs-0.1.2 → lazycogs-0.2.0}/src/lazycogs/_explain.py +12 -7
- lazycogs-0.1.2/src/lazycogs/_executor.py +0 -57
- {lazycogs-0.1.2 → lazycogs-0.2.0}/LICENSE +0 -0
- {lazycogs-0.1.2 → lazycogs-0.2.0}/src/lazycogs/_cql2.py +0 -0
- {lazycogs-0.1.2 → lazycogs-0.2.0}/src/lazycogs/_grid.py +0 -0
- {lazycogs-0.1.2 → lazycogs-0.2.0}/src/lazycogs/_mosaic_methods.py +0 -0
- {lazycogs-0.1.2 → lazycogs-0.2.0}/src/lazycogs/_reproject.py +0 -0
- {lazycogs-0.1.2 → lazycogs-0.2.0}/src/lazycogs/_storage_ext.py +0 -0
- {lazycogs-0.1.2 → lazycogs-0.2.0}/src/lazycogs/_store.py +0 -0
- {lazycogs-0.1.2 → lazycogs-0.2.0}/src/lazycogs/_temporal.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.3
|
|
2
2
|
Name: lazycogs
|
|
3
|
-
Version: 0.
|
|
3
|
+
Version: 0.2.0
|
|
4
4
|
Summary: Lazily load COG assets from STAC items into xarray DataArrays using async-geotiff
|
|
5
5
|
Keywords: COG,STAC,xarray,geotiff,raster,geospatial
|
|
6
6
|
Author: Henry Rodman
|
|
@@ -58,7 +58,7 @@ Description-Content-Type: text/markdown
|
|
|
58
58
|
[](https://pypi.org/project/lazycogs/)
|
|
59
59
|
[](https://github.com/developmentseed/lazycogs/blob/main/LICENSE)
|
|
60
60
|
|
|
61
|
-
Open a lazy `(
|
|
61
|
+
Open a lazy `(band, time, y, x)` xarray DataArray from thousands of cloud-optimized geotiffs (COGs). No GDAL required.
|
|
62
62
|
|
|
63
63
|
## What is lazycogs?
|
|
64
64
|
|
|
@@ -98,7 +98,7 @@ dst_crs = "EPSG:32615"
|
|
|
98
98
|
dst_bbox = (380000.0, 4928000.0, 420000.0, 4984000.0)
|
|
99
99
|
|
|
100
100
|
# transform to 4326 for STAC search
|
|
101
|
-
transformer = Transformer.from_crs(dst_crs, "epsg:4326",
|
|
101
|
+
transformer = Transformer.from_crs(dst_crs, "epsg:4326", always_xy=True)
|
|
102
102
|
bbox_4326 = transformer.transform_bounds(*dst_bbox)
|
|
103
103
|
|
|
104
104
|
# Search a STAC API and cache results to a local stac-geoparquet file.
|
|
@@ -110,7 +110,7 @@ await rustac.search_to(
|
|
|
110
110
|
bbox=bbox_4326,
|
|
111
111
|
)
|
|
112
112
|
|
|
113
|
-
# Open a fully lazy (
|
|
113
|
+
# Open a fully lazy (band, time, y, x) DataArray. No COGs are read yet.
|
|
114
114
|
da = lazycogs.open(
|
|
115
115
|
"items.parquet",
|
|
116
116
|
bbox=dst_bbox,
|
|
@@ -5,7 +5,7 @@
|
|
|
5
5
|
[](https://pypi.org/project/lazycogs/)
|
|
6
6
|
[](https://github.com/developmentseed/lazycogs/blob/main/LICENSE)
|
|
7
7
|
|
|
8
|
-
Open a lazy `(
|
|
8
|
+
Open a lazy `(band, time, y, x)` xarray DataArray from thousands of cloud-optimized geotiffs (COGs). No GDAL required.
|
|
9
9
|
|
|
10
10
|
## What is lazycogs?
|
|
11
11
|
|
|
@@ -45,7 +45,7 @@ dst_crs = "EPSG:32615"
|
|
|
45
45
|
dst_bbox = (380000.0, 4928000.0, 420000.0, 4984000.0)
|
|
46
46
|
|
|
47
47
|
# transform to 4326 for STAC search
|
|
48
|
-
transformer = Transformer.from_crs(dst_crs, "epsg:4326",
|
|
48
|
+
transformer = Transformer.from_crs(dst_crs, "epsg:4326", always_xy=True)
|
|
49
49
|
bbox_4326 = transformer.transform_bounds(*dst_bbox)
|
|
50
50
|
|
|
51
51
|
# Search a STAC API and cache results to a local stac-geoparquet file.
|
|
@@ -57,7 +57,7 @@ await rustac.search_to(
|
|
|
57
57
|
bbox=bbox_4326,
|
|
58
58
|
)
|
|
59
59
|
|
|
60
|
-
# Open a fully lazy (
|
|
60
|
+
# Open a fully lazy (band, time, y, x) DataArray. No COGs are read yet.
|
|
61
61
|
da = lazycogs.open(
|
|
62
62
|
"items.parquet",
|
|
63
63
|
bbox=dst_bbox,
|
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
"""lazycogs: lazy xarray DataArrays from STAC COG collections."""
|
|
2
2
|
|
|
3
|
+
from lazycogs._chunk_reader import read_chunk, read_chunk_async
|
|
3
4
|
from lazycogs._core import open # noqa: A004
|
|
4
5
|
from lazycogs._executor import set_reproject_workers
|
|
5
6
|
from lazycogs._explain import ( # noqa: F401 — registers da.stac_cog accessor
|
|
@@ -35,6 +36,8 @@ __all__ = [
|
|
|
35
36
|
"StdevMethod",
|
|
36
37
|
"align_bbox",
|
|
37
38
|
"open",
|
|
39
|
+
"read_chunk",
|
|
40
|
+
"read_chunk_async",
|
|
38
41
|
"set_reproject_workers",
|
|
39
42
|
"store_for",
|
|
40
43
|
]
|
|
@@ -3,9 +3,7 @@
|
|
|
3
3
|
from __future__ import annotations
|
|
4
4
|
|
|
5
5
|
import asyncio
|
|
6
|
-
import concurrent.futures
|
|
7
6
|
import logging
|
|
8
|
-
import threading
|
|
9
7
|
import time
|
|
10
8
|
from dataclasses import dataclass, field
|
|
11
9
|
from typing import TYPE_CHECKING, Any
|
|
@@ -16,19 +14,18 @@ from pyproj import CRS, Transformer
|
|
|
16
14
|
from xarray.backends.common import BackendArray
|
|
17
15
|
from xarray.core import indexing
|
|
18
16
|
|
|
19
|
-
from lazycogs._chunk_reader import
|
|
17
|
+
from lazycogs._chunk_reader import read_chunk_async
|
|
20
18
|
from lazycogs._cql2 import _extract_filter_fields, _sortby_fields
|
|
21
|
-
from lazycogs._executor import
|
|
19
|
+
from lazycogs._executor import (
|
|
20
|
+
_DUCKDB_EXECUTOR,
|
|
21
|
+
_run_coroutine,
|
|
22
|
+
)
|
|
22
23
|
|
|
23
24
|
logger = logging.getLogger(__name__)
|
|
24
25
|
|
|
25
26
|
|
|
26
|
-
_tls = threading.local()
|
|
27
|
-
|
|
28
27
|
if TYPE_CHECKING:
|
|
29
|
-
from asyncio.futures import Future
|
|
30
28
|
from collections.abc import Callable
|
|
31
|
-
from types import CoroutineType
|
|
32
29
|
|
|
33
30
|
from rustac import DuckdbClient
|
|
34
31
|
|
|
@@ -39,8 +36,8 @@ if TYPE_CHECKING:
|
|
|
39
36
|
class _ChunkReadPlan:
|
|
40
37
|
"""Everything needed to materialise one chunk across all its time steps.
|
|
41
38
|
|
|
42
|
-
Built once in ``
|
|
43
|
-
``
|
|
39
|
+
Built once in ``_async_getitem`` and passed through to
|
|
40
|
+
``_read_chunk_all_dates`` and ``_run_one_date``. Frozen to make the
|
|
44
41
|
read-only intent explicit.
|
|
45
42
|
|
|
46
43
|
Note: ``warp_cache`` is a mutable dict despite the frozen dataclass. This
|
|
@@ -51,7 +48,6 @@ class _ChunkReadPlan:
|
|
|
51
48
|
Attributes:
|
|
52
49
|
duckdb_client: ``DuckdbClient`` instance used for STAC queries.
|
|
53
50
|
parquet_path: Path to the geoparquet file or hive-partitioned directory.
|
|
54
|
-
duckdb_lock: Lock serialising access to ``duckdb_client``.
|
|
55
51
|
sortby: Optional sort keys forwarded to ``client.search``.
|
|
56
52
|
filter_expr: Optional CQL2 filter forwarded to ``client.search``.
|
|
57
53
|
ids: Optional STAC item IDs forwarded to ``client.search``.
|
|
@@ -74,7 +70,6 @@ class _ChunkReadPlan:
|
|
|
74
70
|
|
|
75
71
|
duckdb_client: DuckdbClient
|
|
76
72
|
parquet_path: str
|
|
77
|
-
duckdb_lock: threading.Lock
|
|
78
73
|
sortby: str | list[str | dict[str, str]] | None
|
|
79
74
|
filter_expr: str | dict[str, Any] | None
|
|
80
75
|
ids: list[str] | None
|
|
@@ -94,58 +89,6 @@ class _ChunkReadPlan:
|
|
|
94
89
|
path_fn: Callable[[str], str] | None
|
|
95
90
|
|
|
96
91
|
|
|
97
|
-
def _get_or_create_background_loop() -> asyncio.AbstractEventLoop:
|
|
98
|
-
"""Return the persistent background event loop for the current thread.
|
|
99
|
-
|
|
100
|
-
Creates the loop, its bounded reprojection executor, and its daemon runner
|
|
101
|
-
thread on first call from a given thread. Subsequent calls on the same
|
|
102
|
-
thread return the cached loop immediately.
|
|
103
|
-
|
|
104
|
-
The loop is stored on ``threading.local`` so each thread (each dask worker,
|
|
105
|
-
each Jupyter kernel callback thread) has its own independent loop and
|
|
106
|
-
executor — tasks on different threads do not share a pool.
|
|
107
|
-
|
|
108
|
-
Using a persistent loop (rather than a fresh one per call) ensures that
|
|
109
|
-
any in-flight callbacks from ``async_geotiff``/``obstore`` background
|
|
110
|
-
threads can always be delivered, avoiding ``RuntimeError: Event loop is
|
|
111
|
-
closed`` errors from callbacks that fire after a fresh loop is torn down.
|
|
112
|
-
"""
|
|
113
|
-
loop: asyncio.AbstractEventLoop | None = getattr(_tls, "loop", None)
|
|
114
|
-
if loop is not None and loop.is_running():
|
|
115
|
-
return loop
|
|
116
|
-
|
|
117
|
-
loop = asyncio.new_event_loop()
|
|
118
|
-
loop.set_default_executor(
|
|
119
|
-
concurrent.futures.ThreadPoolExecutor(
|
|
120
|
-
max_workers=get_max_workers(),
|
|
121
|
-
thread_name_prefix="lazycogs-reproject",
|
|
122
|
-
),
|
|
123
|
-
)
|
|
124
|
-
t = threading.Thread(target=loop.run_forever, daemon=True, name="lazycogs-loop")
|
|
125
|
-
t.start()
|
|
126
|
-
_tls.loop = loop
|
|
127
|
-
return loop
|
|
128
|
-
|
|
129
|
-
|
|
130
|
-
def _run_coroutine(coro: CoroutineType) -> Future:
|
|
131
|
-
"""Run an async coroutine from sync code.
|
|
132
|
-
|
|
133
|
-
Submits the coroutine to a persistent per-thread background event loop,
|
|
134
|
-
blocking until it completes. The background loop is created on the first
|
|
135
|
-
call from a given thread (dask worker, Jupyter kernel thread, etc.) and
|
|
136
|
-
reused for all subsequent calls on that same thread.
|
|
137
|
-
|
|
138
|
-
Args:
|
|
139
|
-
coro: The coroutine to execute.
|
|
140
|
-
|
|
141
|
-
Returns:
|
|
142
|
-
The return value of the coroutine.
|
|
143
|
-
|
|
144
|
-
"""
|
|
145
|
-
loop = _get_or_create_background_loop()
|
|
146
|
-
return asyncio.run_coroutine_threadsafe(coro, loop).result()
|
|
147
|
-
|
|
148
|
-
|
|
149
92
|
@dataclass
|
|
150
93
|
class _SpatialWindow:
|
|
151
94
|
"""Resolved spatial indexing window.
|
|
@@ -216,7 +159,7 @@ def _resolve_band_indices(
|
|
|
216
159
|
return list(range(start, stop, step)), False
|
|
217
160
|
|
|
218
161
|
|
|
219
|
-
def
|
|
162
|
+
def _search_items_sync(
|
|
220
163
|
plan: _ChunkReadPlan,
|
|
221
164
|
date: str,
|
|
222
165
|
) -> list[Any]:
|
|
@@ -261,6 +204,28 @@ def _search_items(
|
|
|
261
204
|
return items
|
|
262
205
|
|
|
263
206
|
|
|
207
|
+
async def _search_items_async(
|
|
208
|
+
plan: _ChunkReadPlan,
|
|
209
|
+
date: str,
|
|
210
|
+
) -> list[Any]:
|
|
211
|
+
"""Run the DuckDB search on the dedicated DuckDB executor.
|
|
212
|
+
|
|
213
|
+
DuckDB queries serialise on a single connection internally, so this
|
|
214
|
+
yields the event loop during the query but does not produce parallel
|
|
215
|
+
queries against the same DuckdbClient.
|
|
216
|
+
|
|
217
|
+
Args:
|
|
218
|
+
plan: Read plan carrying all parameters for this chunk.
|
|
219
|
+
date: Acquisition date string (``"YYYY-MM-DD"``).
|
|
220
|
+
|
|
221
|
+
Returns:
|
|
222
|
+
List of STAC items returned by DuckDB.
|
|
223
|
+
|
|
224
|
+
"""
|
|
225
|
+
loop = asyncio.get_running_loop()
|
|
226
|
+
return await loop.run_in_executor(_DUCKDB_EXECUTOR, _search_items_sync, plan, date)
|
|
227
|
+
|
|
228
|
+
|
|
264
229
|
async def _run_one_date(
|
|
265
230
|
t_idx: int,
|
|
266
231
|
plan: _ChunkReadPlan,
|
|
@@ -268,7 +233,7 @@ async def _run_one_date(
|
|
|
268
233
|
"""Read and mosaic all COGs for a single time step.
|
|
269
234
|
|
|
270
235
|
Issues one DuckDB query for items overlapping the chunk at this date, then
|
|
271
|
-
calls
|
|
236
|
+
calls read_chunk_async to fetch and reproject all tiles.
|
|
272
237
|
Returns None if no items match the query.
|
|
273
238
|
|
|
274
239
|
Args:
|
|
@@ -280,14 +245,12 @@ async def _run_one_date(
|
|
|
280
245
|
|
|
281
246
|
"""
|
|
282
247
|
date = plan.dates[t_idx]
|
|
283
|
-
|
|
284
|
-
with plan.duckdb_lock:
|
|
285
|
-
items = _search_items(plan, date)
|
|
248
|
+
items = await _search_items_async(plan, date)
|
|
286
249
|
|
|
287
250
|
if not items:
|
|
288
251
|
return None
|
|
289
252
|
t0 = time.perf_counter()
|
|
290
|
-
chunk_result = await
|
|
253
|
+
chunk_result = await read_chunk_async(
|
|
291
254
|
items=items,
|
|
292
255
|
bands=plan.selected_bands,
|
|
293
256
|
chunk_affine=plan.chunk_affine,
|
|
@@ -302,7 +265,7 @@ async def _run_one_date(
|
|
|
302
265
|
path_fn=plan.path_fn,
|
|
303
266
|
)
|
|
304
267
|
logger.debug(
|
|
305
|
-
"
|
|
268
|
+
"read_chunk_async bands=%r date=%s (%d items, %dx%d px) took %.3fs",
|
|
306
269
|
plan.selected_bands,
|
|
307
270
|
date,
|
|
308
271
|
len(items),
|
|
@@ -313,15 +276,17 @@ async def _run_one_date(
|
|
|
313
276
|
return chunk_result
|
|
314
277
|
|
|
315
278
|
|
|
316
|
-
async def
|
|
279
|
+
async def _read_chunk_all_dates(
|
|
317
280
|
time_indices: list[int],
|
|
318
281
|
plan: _ChunkReadPlan,
|
|
319
282
|
) -> list[dict[str, np.ndarray] | None]:
|
|
320
283
|
"""Run all time steps concurrently inside a single event loop.
|
|
321
284
|
|
|
322
|
-
DuckDB queries
|
|
323
|
-
serialises access
|
|
324
|
-
|
|
285
|
+
DuckDB queries run on the dedicated DuckDB executor; DuckDB itself
|
|
286
|
+
serialises access on a single connection, so concurrent queries on the
|
|
287
|
+
same ``DuckdbClient`` are safe but not parallel. Mosaic coroutines for
|
|
288
|
+
all time steps are gathered concurrently so COG reads and reprojections
|
|
289
|
+
overlap across time steps.
|
|
325
290
|
|
|
326
291
|
Args:
|
|
327
292
|
time_indices: Ordered list of time-dimension indices to materialise.
|
|
@@ -409,12 +374,6 @@ class MultiBandStacBackendArray(BackendArray):
|
|
|
409
374
|
path_from_href: Callable[[str], str] | None = field(default=None)
|
|
410
375
|
shape: tuple[int, ...] = field(init=False)
|
|
411
376
|
_dst_to_4326: Transformer | None = field(init=False, repr=False, compare=False)
|
|
412
|
-
_duckdb_lock: threading.Lock = field(
|
|
413
|
-
init=False,
|
|
414
|
-
repr=False,
|
|
415
|
-
compare=False,
|
|
416
|
-
default_factory=threading.Lock,
|
|
417
|
-
)
|
|
418
377
|
|
|
419
378
|
def __post_init__(self) -> None:
|
|
420
379
|
"""Derive shape and cache the dst→EPSG:4326 transformer."""
|
|
@@ -429,11 +388,6 @@ class MultiBandStacBackendArray(BackendArray):
|
|
|
429
388
|
always_xy=True,
|
|
430
389
|
)
|
|
431
390
|
|
|
432
|
-
@property
|
|
433
|
-
def duckdb_lock(self) -> threading.Lock:
|
|
434
|
-
"""Lock serialising access to ``duckdb_client``."""
|
|
435
|
-
return self._duckdb_lock
|
|
436
|
-
|
|
437
391
|
def __repr__(self) -> str:
|
|
438
392
|
"""Return a compact string representation."""
|
|
439
393
|
return f"MultiBandStacBackendArray(bands={self.bands!r}, shape={self.shape})"
|
|
@@ -529,16 +483,30 @@ class MultiBandStacBackendArray(BackendArray):
|
|
|
529
483
|
key,
|
|
530
484
|
self.shape,
|
|
531
485
|
indexing.IndexingSupport.BASIC,
|
|
532
|
-
self.
|
|
486
|
+
self._sync_getitem,
|
|
533
487
|
)
|
|
534
488
|
|
|
535
|
-
def
|
|
489
|
+
def _sync_getitem(self, key: tuple[Any, ...]) -> np.ndarray:
|
|
490
|
+
"""Sync adapter that runs ``_async_getitem`` on the background loop.
|
|
491
|
+
|
|
492
|
+
Args:
|
|
493
|
+
key: A tuple of ``int | slice`` objects for the
|
|
494
|
+
``(band, time, y, x)`` dimensions.
|
|
495
|
+
|
|
496
|
+
Returns:
|
|
497
|
+
Numpy array with shape determined by the indexing key.
|
|
498
|
+
|
|
499
|
+
"""
|
|
500
|
+
return _run_coroutine(self._async_getitem(key))
|
|
501
|
+
|
|
502
|
+
async def _async_getitem(self, key: tuple[Any, ...]) -> np.ndarray:
|
|
536
503
|
"""Materialise the chunk identified by ``key``.
|
|
537
504
|
|
|
538
|
-
|
|
539
|
-
|
|
540
|
-
a single
|
|
541
|
-
|
|
505
|
+
Single source of truth for chunk reads. Reads all selected bands
|
|
506
|
+
together per time step via
|
|
507
|
+
:func:`~lazycogs._chunk_reader.read_chunk_async`, issuing a single
|
|
508
|
+
DuckDB query per time step and sharing reprojection warp maps across
|
|
509
|
+
bands that have identical source geometry.
|
|
542
510
|
|
|
543
511
|
Args:
|
|
544
512
|
key: A tuple of ``int | slice`` objects for the
|
|
@@ -562,7 +530,6 @@ class MultiBandStacBackendArray(BackendArray):
|
|
|
562
530
|
plan = _ChunkReadPlan(
|
|
563
531
|
duckdb_client=self.duckdb_client,
|
|
564
532
|
parquet_path=self.parquet_path,
|
|
565
|
-
duckdb_lock=self._duckdb_lock,
|
|
566
533
|
sortby=self.sortby,
|
|
567
534
|
filter_expr=self.filter,
|
|
568
535
|
ids=self.ids,
|
|
@@ -582,7 +549,7 @@ class MultiBandStacBackendArray(BackendArray):
|
|
|
582
549
|
path_fn=self.path_from_href,
|
|
583
550
|
)
|
|
584
551
|
|
|
585
|
-
all_chunk_data =
|
|
552
|
+
all_chunk_data = await _read_chunk_all_dates(time_indices, plan)
|
|
586
553
|
|
|
587
554
|
out_shape = (
|
|
588
555
|
len(band_indices),
|
|
@@ -12,6 +12,7 @@ import numpy as np
|
|
|
12
12
|
from async_geotiff import GeoTIFF, Overview, RasterArray, Window
|
|
13
13
|
from numpy import ma
|
|
14
14
|
|
|
15
|
+
from lazycogs._executor import _run_coroutine
|
|
15
16
|
from lazycogs._mosaic_methods import FirstMethod, MosaicMethodBase
|
|
16
17
|
from lazycogs._reproject import (
|
|
17
18
|
WarpMap,
|
|
@@ -510,7 +511,7 @@ async def _drain_in_order(
|
|
|
510
511
|
await asyncio.gather(*tasks, return_exceptions=True)
|
|
511
512
|
|
|
512
513
|
|
|
513
|
-
async def
|
|
514
|
+
async def read_chunk_async(
|
|
514
515
|
items: list[dict],
|
|
515
516
|
bands: list[str],
|
|
516
517
|
chunk_affine: Affine,
|
|
@@ -527,8 +528,7 @@ async def async_mosaic_chunk(
|
|
|
527
528
|
"""Read, reproject, and mosaic multiple bands from a list of STAC items.
|
|
528
529
|
|
|
529
530
|
Processes all requested bands together per item so that bands sharing the
|
|
530
|
-
same source geometry
|
|
531
|
-
compute the reprojection warp map only once (via
|
|
531
|
+
same source geometry compute the reprojection warp map only once (via
|
|
532
532
|
:func:`_apply_bands_with_warp_cache`).
|
|
533
533
|
|
|
534
534
|
Items are processed in batches of ``max_concurrent_reads``. When all
|
|
@@ -615,3 +615,62 @@ async def async_mosaic_chunk(
|
|
|
615
615
|
dtype=np.float32,
|
|
616
616
|
)
|
|
617
617
|
return output
|
|
618
|
+
|
|
619
|
+
|
|
620
|
+
def read_chunk(
|
|
621
|
+
items: list[dict],
|
|
622
|
+
bands: list[str],
|
|
623
|
+
chunk_affine: Affine,
|
|
624
|
+
dst_crs: CRS,
|
|
625
|
+
chunk_width: int,
|
|
626
|
+
chunk_height: int,
|
|
627
|
+
nodata: float | None = None,
|
|
628
|
+
mosaic_method_cls: type[MosaicMethodBase] | None = None,
|
|
629
|
+
store: ObjectStore | None = None,
|
|
630
|
+
max_concurrent_reads: int = 32,
|
|
631
|
+
warp_cache: dict | None = None,
|
|
632
|
+
path_fn: Callable[[str], str] | None = None,
|
|
633
|
+
) -> dict[str, np.ndarray]:
|
|
634
|
+
"""Run :func:`read_chunk_async` on the persistent per-thread background loop.
|
|
635
|
+
|
|
636
|
+
All arguments are identical to :func:`read_chunk_async`.
|
|
637
|
+
|
|
638
|
+
Args:
|
|
639
|
+
items: List of STAC item dicts to mosaic. Processed in order.
|
|
640
|
+
bands: Asset keys identifying the bands to read from each item.
|
|
641
|
+
chunk_affine: Affine transform of the destination chunk.
|
|
642
|
+
dst_crs: CRS of the destination chunk.
|
|
643
|
+
chunk_width: Width of the destination chunk in pixels.
|
|
644
|
+
chunk_height: Height of the destination chunk in pixels.
|
|
645
|
+
nodata: No-data fill value.
|
|
646
|
+
mosaic_method_cls: Mosaic method class instantiated once per band.
|
|
647
|
+
Defaults to :class:`~lazycogs._mosaic_methods.FirstMethod`.
|
|
648
|
+
store: Optional pre-configured obstore ``ObjectStore`` instance.
|
|
649
|
+
max_concurrent_reads: Maximum number of COG reads to run concurrently.
|
|
650
|
+
warp_cache: Optional cache shared across calls for reusing warp maps
|
|
651
|
+
from earlier time steps.
|
|
652
|
+
path_fn: Optional callable that takes an asset HREF and returns the
|
|
653
|
+
object path to use with *store*.
|
|
654
|
+
|
|
655
|
+
Returns:
|
|
656
|
+
``dict`` mapping each band name to an array of shape
|
|
657
|
+
``(cog_bands, chunk_height, chunk_width)`` with dtype matching the
|
|
658
|
+
source COGs.
|
|
659
|
+
|
|
660
|
+
"""
|
|
661
|
+
return _run_coroutine(
|
|
662
|
+
read_chunk_async(
|
|
663
|
+
items=items,
|
|
664
|
+
bands=bands,
|
|
665
|
+
chunk_affine=chunk_affine,
|
|
666
|
+
dst_crs=dst_crs,
|
|
667
|
+
chunk_width=chunk_width,
|
|
668
|
+
chunk_height=chunk_height,
|
|
669
|
+
nodata=nodata,
|
|
670
|
+
mosaic_method_cls=mosaic_method_cls,
|
|
671
|
+
store=store,
|
|
672
|
+
max_concurrent_reads=max_concurrent_reads,
|
|
673
|
+
warp_cache=warp_cache,
|
|
674
|
+
path_fn=path_fn,
|
|
675
|
+
),
|
|
676
|
+
)
|
|
@@ -308,7 +308,7 @@ def _build_dataarray(
|
|
|
308
308
|
:func:`open` for full documentation.
|
|
309
309
|
|
|
310
310
|
Returns:
|
|
311
|
-
Lazy ``xr.DataArray`` with dimensions ``(
|
|
311
|
+
Lazy ``xr.DataArray`` with dimensions ``(band, time, y, x)``.
|
|
312
312
|
|
|
313
313
|
"""
|
|
314
314
|
dst_affine, dst_width, dst_height, x_coords, y_coords = compute_output_grid(
|
|
@@ -386,7 +386,7 @@ def open( # noqa: A001
|
|
|
386
386
|
path_from_href: Callable[[str], str] | None = None,
|
|
387
387
|
duckdb_client: DuckdbClient | None = None,
|
|
388
388
|
) -> xr.DataArray:
|
|
389
|
-
"""Open a mosaic of STAC items as a lazy ``(
|
|
389
|
+
"""Open a mosaic of STAC items as a lazy ``(band, time, y, x)`` DataArray.
|
|
390
390
|
|
|
391
391
|
``href`` must be a path to a geoparquet file (``.parquet`` or
|
|
392
392
|
``.geoparquet``) or, when *duckdb_client* is provided, to a
|
|
@@ -480,7 +480,7 @@ def open( # noqa: A001
|
|
|
480
480
|
)
|
|
481
481
|
|
|
482
482
|
Returns:
|
|
483
|
-
Lazy ``xr.DataArray`` with dimensions ``(
|
|
483
|
+
Lazy ``xr.DataArray`` with dimensions ``(band, time, y, x)``.
|
|
484
484
|
|
|
485
485
|
Raises:
|
|
486
486
|
ValueError: If ``href`` is not a ``.parquet`` or ``.geoparquet`` file
|
|
@@ -0,0 +1,128 @@
|
|
|
1
|
+
"""Thread pool and event loop configuration for reprojection and DuckDB work."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import asyncio
|
|
6
|
+
import concurrent.futures
|
|
7
|
+
import os
|
|
8
|
+
import threading
|
|
9
|
+
from typing import TYPE_CHECKING
|
|
10
|
+
|
|
11
|
+
if TYPE_CHECKING:
|
|
12
|
+
from asyncio.futures import Future
|
|
13
|
+
from types import CoroutineType
|
|
14
|
+
|
|
15
|
+
config: dict[str, int | None] = {"max_workers": None}
|
|
16
|
+
|
|
17
|
+
_tls = threading.local()
|
|
18
|
+
|
|
19
|
+
# DuckDB queries serialise on a single connection, so a small pool is enough.
|
|
20
|
+
# Kept separate from the reprojection executor so a long reprojection cannot
|
|
21
|
+
# starve a queued query within the same chunk read.
|
|
22
|
+
_DUCKDB_EXECUTOR = concurrent.futures.ThreadPoolExecutor(
|
|
23
|
+
max_workers=2,
|
|
24
|
+
thread_name_prefix="lazycogs-duckdb",
|
|
25
|
+
)
|
|
26
|
+
|
|
27
|
+
|
|
28
|
+
def _default_workers() -> int:
|
|
29
|
+
"""Return the default worker count: CPUs up to a cap of 4.
|
|
30
|
+
|
|
31
|
+
Reprojection (pyproj + numpy) is memory-bandwidth-bound, not compute-bound.
|
|
32
|
+
Benchmarks show diminishing returns beyond 4 concurrent threads because they
|
|
33
|
+
saturate the memory bus rather than adding CPU throughput. Keep the default
|
|
34
|
+
conservative.
|
|
35
|
+
"""
|
|
36
|
+
return min(os.cpu_count() or 4, 4)
|
|
37
|
+
|
|
38
|
+
|
|
39
|
+
def get_max_workers() -> int:
|
|
40
|
+
"""Return the configured worker count, or the default if not set.
|
|
41
|
+
|
|
42
|
+
Returns:
|
|
43
|
+
Number of reprojection threads each event loop will use.
|
|
44
|
+
|
|
45
|
+
"""
|
|
46
|
+
val = config["max_workers"]
|
|
47
|
+
return val if val is not None else _default_workers()
|
|
48
|
+
|
|
49
|
+
|
|
50
|
+
def set_reproject_workers(n: int) -> None:
|
|
51
|
+
"""Set the number of threads each thread's event loop uses for reprojection.
|
|
52
|
+
|
|
53
|
+
Each thread (dask worker, Jupyter kernel callback thread, etc.) gets one
|
|
54
|
+
persistent background event loop with one bounded reprojection
|
|
55
|
+
``ThreadPoolExecutor``. All chunk reads on that thread share the same loop
|
|
56
|
+
and executor. Dask tasks on different threads do not compete for a shared
|
|
57
|
+
pool. Total reprojection threads at any moment is at most
|
|
58
|
+
``n x active_thread_count``.
|
|
59
|
+
|
|
60
|
+
Reprojection is memory-bandwidth-bound rather than compute-bound, so values
|
|
61
|
+
above 4 typically offer no benefit and can hurt throughput due to memory
|
|
62
|
+
contention. The default is ``min(os.cpu_count(), 4)``.
|
|
63
|
+
|
|
64
|
+
To improve overall throughput, prefer adding time or band parallelism via
|
|
65
|
+
dask (``chunks={"time": 1}``) over raising this value.
|
|
66
|
+
|
|
67
|
+
Args:
|
|
68
|
+
n: Number of worker threads per event loop. Must be >= 1.
|
|
69
|
+
|
|
70
|
+
Raises:
|
|
71
|
+
ValueError: If ``n`` is less than 1.
|
|
72
|
+
|
|
73
|
+
"""
|
|
74
|
+
if n < 1:
|
|
75
|
+
raise ValueError(f"n must be >= 1, got {n!r}")
|
|
76
|
+
config["max_workers"] = n
|
|
77
|
+
|
|
78
|
+
|
|
79
|
+
def _get_or_create_background_loop() -> asyncio.AbstractEventLoop:
|
|
80
|
+
"""Return the persistent background event loop for the current thread.
|
|
81
|
+
|
|
82
|
+
Creates the loop, its bounded reprojection executor, and its daemon runner
|
|
83
|
+
thread on first call from a given thread. Subsequent calls on the same
|
|
84
|
+
thread return the cached loop immediately.
|
|
85
|
+
|
|
86
|
+
The loop is stored on ``threading.local`` so each thread (each dask worker,
|
|
87
|
+
each Jupyter kernel callback thread) has its own independent loop and
|
|
88
|
+
executor — tasks on different threads do not share a pool.
|
|
89
|
+
|
|
90
|
+
Using a persistent loop (rather than a fresh one per call) ensures that
|
|
91
|
+
any in-flight callbacks from ``async_geotiff``/``obstore`` background
|
|
92
|
+
threads can always be delivered, avoiding ``RuntimeError: Event loop is
|
|
93
|
+
closed`` errors from callbacks that fire after a fresh loop is torn down.
|
|
94
|
+
"""
|
|
95
|
+
loop: asyncio.AbstractEventLoop | None = getattr(_tls, "loop", None)
|
|
96
|
+
if loop is not None and loop.is_running():
|
|
97
|
+
return loop
|
|
98
|
+
|
|
99
|
+
loop = asyncio.new_event_loop()
|
|
100
|
+
loop.set_default_executor(
|
|
101
|
+
concurrent.futures.ThreadPoolExecutor(
|
|
102
|
+
max_workers=get_max_workers(),
|
|
103
|
+
thread_name_prefix="lazycogs-reproject",
|
|
104
|
+
),
|
|
105
|
+
)
|
|
106
|
+
t = threading.Thread(target=loop.run_forever, daemon=True, name="lazycogs-loop")
|
|
107
|
+
t.start()
|
|
108
|
+
_tls.loop = loop
|
|
109
|
+
return loop
|
|
110
|
+
|
|
111
|
+
|
|
112
|
+
def _run_coroutine(coro: CoroutineType) -> Future:
|
|
113
|
+
"""Run an async coroutine from sync code.
|
|
114
|
+
|
|
115
|
+
Submits the coroutine to a persistent per-thread background event loop,
|
|
116
|
+
blocking until it completes. The background loop is created on the first
|
|
117
|
+
call from a given thread (dask worker, Jupyter kernel thread, etc.) and
|
|
118
|
+
reused for all subsequent calls on that same thread.
|
|
119
|
+
|
|
120
|
+
Args:
|
|
121
|
+
coro: The coroutine to execute.
|
|
122
|
+
|
|
123
|
+
Returns:
|
|
124
|
+
The return value of the coroutine.
|
|
125
|
+
|
|
126
|
+
"""
|
|
127
|
+
loop = _get_or_create_background_loop()
|
|
128
|
+
return asyncio.run_coroutine_threadsafe(coro, loop).result()
|
|
@@ -14,14 +14,16 @@ from affine import Affine
|
|
|
14
14
|
from pandas import DataFrame
|
|
15
15
|
from pyproj import CRS, Transformer
|
|
16
16
|
|
|
17
|
-
from lazycogs._backend import MultiBandStacBackendArray, _run_coroutine
|
|
18
17
|
from lazycogs._chunk_reader import _ChunkContext, _open_and_window
|
|
18
|
+
from lazycogs._executor import _DUCKDB_EXECUTOR, _run_coroutine
|
|
19
19
|
|
|
20
20
|
if TYPE_CHECKING:
|
|
21
21
|
from collections.abc import Iterator
|
|
22
22
|
|
|
23
23
|
from obstore.store import ObjectStore
|
|
24
24
|
|
|
25
|
+
from lazycogs._backend import MultiBandStacBackendArray
|
|
26
|
+
|
|
25
27
|
logger = logging.getLogger(__name__)
|
|
26
28
|
|
|
27
29
|
_COG_MULTI_THRESHOLD = 2
|
|
@@ -505,9 +507,9 @@ async def _explain_async(
|
|
|
505
507
|
Issues one DuckDB query per ``(time step, spatial tile)`` — not one per
|
|
506
508
|
``(band, time step, spatial tile)`` — because the query result is
|
|
507
509
|
band-independent. All ``(time x tile)`` queries are dispatched
|
|
508
|
-
concurrently via :func:`asyncio.gather
|
|
509
|
-
|
|
510
|
-
all active bands to produce one :class:`ChunkRead` per
|
|
510
|
+
concurrently via :func:`asyncio.gather` so the per-query JSON
|
|
511
|
+
construction and result processing overlap. Each query result is then
|
|
512
|
+
fanned across all active bands to produce one :class:`ChunkRead` per
|
|
511
513
|
``(band, time, tile)`` combination.
|
|
512
514
|
|
|
513
515
|
Args:
|
|
@@ -598,15 +600,18 @@ async def _explain_async(
|
|
|
598
600
|
actual_h,
|
|
599
601
|
dst_crs,
|
|
600
602
|
)
|
|
601
|
-
|
|
602
|
-
|
|
603
|
+
loop = asyncio.get_running_loop()
|
|
604
|
+
items = await loop.run_in_executor(
|
|
605
|
+
_DUCKDB_EXECUTOR,
|
|
606
|
+
lambda: backend.duckdb_client.search(
|
|
603
607
|
backend.parquet_path,
|
|
604
608
|
bbox=chunk_bbox_4326,
|
|
605
609
|
datetime=date_filter,
|
|
606
610
|
sortby=backend.sortby,
|
|
607
611
|
filter=backend.filter,
|
|
608
612
|
ids=backend.ids,
|
|
609
|
-
)
|
|
613
|
+
),
|
|
614
|
+
)
|
|
610
615
|
logger.debug(
|
|
611
616
|
"explain date=%s chunk=(%d,%d) -> %d items (for %d band(s))",
|
|
612
617
|
date_filter,
|
|
@@ -1,57 +0,0 @@
|
|
|
1
|
-
"""Configure the per-event-loop thread pool size for CPU-bound reprojection work."""
|
|
2
|
-
|
|
3
|
-
from __future__ import annotations
|
|
4
|
-
|
|
5
|
-
import os
|
|
6
|
-
|
|
7
|
-
config: dict[str, int | None] = {"max_workers": None}
|
|
8
|
-
|
|
9
|
-
|
|
10
|
-
def _default_workers() -> int:
|
|
11
|
-
"""Return the default worker count: CPUs up to a cap of 4.
|
|
12
|
-
|
|
13
|
-
Reprojection (pyproj + numpy) is memory-bandwidth-bound, not compute-bound.
|
|
14
|
-
Benchmarks show diminishing returns beyond 4 concurrent threads because they
|
|
15
|
-
saturate the memory bus rather than adding CPU throughput. Keep the default
|
|
16
|
-
conservative.
|
|
17
|
-
"""
|
|
18
|
-
return min(os.cpu_count() or 4, 4)
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
def get_max_workers() -> int:
|
|
22
|
-
"""Return the configured worker count, or the default if not set.
|
|
23
|
-
|
|
24
|
-
Returns:
|
|
25
|
-
Number of reprojection threads each event loop will use.
|
|
26
|
-
|
|
27
|
-
"""
|
|
28
|
-
val = config["max_workers"]
|
|
29
|
-
return val if val is not None else _default_workers()
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
def set_reproject_workers(n: int) -> None:
|
|
33
|
-
"""Set the number of threads each chunk's event loop uses for reprojection.
|
|
34
|
-
|
|
35
|
-
Each chunk read creates a fresh asyncio event loop with its own dedicated
|
|
36
|
-
``ThreadPoolExecutor`` bounded to ``n`` workers. Dask tasks do not compete
|
|
37
|
-
for a shared pool — each task gets ``n`` independent reprojection threads.
|
|
38
|
-
Total reprojection threads at any moment is at most
|
|
39
|
-
``n x dask_worker_count``.
|
|
40
|
-
|
|
41
|
-
Reprojection is memory-bandwidth-bound rather than compute-bound, so values
|
|
42
|
-
above 4 typically offer no benefit and can hurt throughput due to memory
|
|
43
|
-
contention. The default is ``min(os.cpu_count(), 4)``.
|
|
44
|
-
|
|
45
|
-
To improve overall throughput, prefer adding time or band parallelism via
|
|
46
|
-
dask (``chunks={"time": 1}``) over raising this value.
|
|
47
|
-
|
|
48
|
-
Args:
|
|
49
|
-
n: Number of worker threads per event loop. Must be >= 1.
|
|
50
|
-
|
|
51
|
-
Raises:
|
|
52
|
-
ValueError: If ``n`` is less than 1.
|
|
53
|
-
|
|
54
|
-
"""
|
|
55
|
-
if n < 1:
|
|
56
|
-
raise ValueError(f"n must be >= 1, got {n!r}")
|
|
57
|
-
config["max_workers"] = n
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|