lazycogs 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- lazycogs/__init__.py +41 -0
- lazycogs/_backend.py +637 -0
- lazycogs/_chunk_reader.py +590 -0
- lazycogs/_core.py +771 -0
- lazycogs/_cql2.py +62 -0
- lazycogs/_executor.py +57 -0
- lazycogs/_explain.py +743 -0
- lazycogs/_grid.py +87 -0
- lazycogs/_mosaic_methods.py +268 -0
- lazycogs/_reproject.py +181 -0
- lazycogs/_storage_ext.py +102 -0
- lazycogs/_store.py +167 -0
- lazycogs/_temporal.py +235 -0
- lazycogs-0.1.0.dist-info/METADATA +121 -0
- lazycogs-0.1.0.dist-info/RECORD +16 -0
- lazycogs-0.1.0.dist-info/WHEEL +4 -0
lazycogs/__init__.py
ADDED
|
@@ -0,0 +1,41 @@
|
|
|
1
|
+
"""lazycogs: lazy xarray DataArrays from STAC COG collections."""
|
|
2
|
+
|
|
3
|
+
from lazycogs._core import open, open_async
|
|
4
|
+
from lazycogs._executor import set_reproject_workers
|
|
5
|
+
from lazycogs._grid import align_bbox
|
|
6
|
+
from lazycogs._store import store_for
|
|
7
|
+
from lazycogs._explain import ( # noqa: F401 — registers da.stac_cog accessor
|
|
8
|
+
ChunkRead,
|
|
9
|
+
ExplainPlan,
|
|
10
|
+
CogRead,
|
|
11
|
+
StacCogAccessor,
|
|
12
|
+
)
|
|
13
|
+
from lazycogs._mosaic_methods import (
|
|
14
|
+
CountMethod,
|
|
15
|
+
FirstMethod,
|
|
16
|
+
HighestMethod,
|
|
17
|
+
LowestMethod,
|
|
18
|
+
MeanMethod,
|
|
19
|
+
MedianMethod,
|
|
20
|
+
MosaicMethodBase,
|
|
21
|
+
StdevMethod,
|
|
22
|
+
)
|
|
23
|
+
|
|
24
|
+
__all__ = [
|
|
25
|
+
"open",
|
|
26
|
+
"open_async",
|
|
27
|
+
"align_bbox",
|
|
28
|
+
"store_for",
|
|
29
|
+
"set_reproject_workers",
|
|
30
|
+
"ExplainPlan",
|
|
31
|
+
"ChunkRead",
|
|
32
|
+
"CogRead",
|
|
33
|
+
"MosaicMethodBase",
|
|
34
|
+
"FirstMethod",
|
|
35
|
+
"HighestMethod",
|
|
36
|
+
"LowestMethod",
|
|
37
|
+
"MeanMethod",
|
|
38
|
+
"MedianMethod",
|
|
39
|
+
"StdevMethod",
|
|
40
|
+
"CountMethod",
|
|
41
|
+
]
|
lazycogs/_backend.py
ADDED
|
@@ -0,0 +1,637 @@
|
|
|
1
|
+
"""xarray BackendArray implementation for lazy STAC COG access."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import asyncio
|
|
6
|
+
import concurrent.futures
|
|
7
|
+
import logging
|
|
8
|
+
import threading
|
|
9
|
+
import time
|
|
10
|
+
from collections.abc import Callable
|
|
11
|
+
from dataclasses import dataclass, field
|
|
12
|
+
from typing import Any
|
|
13
|
+
|
|
14
|
+
import numpy as np
|
|
15
|
+
from affine import Affine
|
|
16
|
+
from pyproj import CRS, Transformer
|
|
17
|
+
from rustac import DuckdbClient
|
|
18
|
+
from xarray.backends.common import BackendArray
|
|
19
|
+
from xarray.core import indexing
|
|
20
|
+
|
|
21
|
+
from lazycogs._chunk_reader import async_mosaic_chunk
|
|
22
|
+
from lazycogs._cql2 import _extract_filter_fields, _sortby_fields
|
|
23
|
+
from lazycogs._executor import get_max_workers
|
|
24
|
+
from lazycogs._mosaic_methods import MosaicMethodBase
|
|
25
|
+
|
|
26
|
+
logger = logging.getLogger(__name__)
|
|
27
|
+
|
|
28
|
+
|
|
29
|
+
_tls = threading.local()
|
|
30
|
+
|
|
31
|
+
|
|
32
|
+
@dataclass(frozen=True)
|
|
33
|
+
class _ChunkReadPlan:
|
|
34
|
+
"""Everything needed to materialise one chunk across all its time steps.
|
|
35
|
+
|
|
36
|
+
Built once in ``_raw_getitem`` and passed through to
|
|
37
|
+
``_run_mosaic_all_dates`` and ``_run_one_date``. Frozen to make the
|
|
38
|
+
read-only intent explicit.
|
|
39
|
+
|
|
40
|
+
Note: ``warp_cache`` is a mutable dict despite the frozen dataclass. This
|
|
41
|
+
is intentional — concurrent writes from ``asyncio.gather`` coroutines are
|
|
42
|
+
safe because ``compute_warp_map`` is deterministic (a duplicate write
|
|
43
|
+
simply overwrites an identical value).
|
|
44
|
+
|
|
45
|
+
Attributes:
|
|
46
|
+
duckdb_client: ``DuckdbClient`` instance used for STAC queries.
|
|
47
|
+
parquet_path: Path to the geoparquet file or hive-partitioned directory.
|
|
48
|
+
duckdb_lock: Lock serialising access to ``duckdb_client``.
|
|
49
|
+
sortby: Optional sort keys forwarded to ``client.search``.
|
|
50
|
+
filter_expr: Optional CQL2 filter forwarded to ``client.search``.
|
|
51
|
+
ids: Optional STAC item IDs forwarded to ``client.search``.
|
|
52
|
+
filter_fields: Field names extracted from ``filter_expr``.
|
|
53
|
+
dates: Full list of acquisition date strings.
|
|
54
|
+
chunk_bbox_4326: ``[minx, miny, maxx, maxy]`` in EPSG:4326.
|
|
55
|
+
selected_bands: STAC asset keys to read.
|
|
56
|
+
chunk_affine: Affine transform of the chunk.
|
|
57
|
+
dst_crs: CRS of the output grid.
|
|
58
|
+
chunk_width: Chunk width in pixels.
|
|
59
|
+
chunk_height: Chunk height in pixels.
|
|
60
|
+
nodata: No-data fill value, or ``None``.
|
|
61
|
+
mosaic_method_cls: Mosaic method class, or ``None`` for the default.
|
|
62
|
+
store: Pre-configured obstore ``ObjectStore`` instance, or ``None``.
|
|
63
|
+
max_concurrent_reads: Maximum concurrent COG reads per chunk.
|
|
64
|
+
warp_cache: Shared warp map cache across time steps.
|
|
65
|
+
path_fn: Optional callable extracting an object path from an asset HREF.
|
|
66
|
+
|
|
67
|
+
"""
|
|
68
|
+
|
|
69
|
+
duckdb_client: DuckdbClient
|
|
70
|
+
parquet_path: str
|
|
71
|
+
duckdb_lock: threading.Lock
|
|
72
|
+
sortby: str | list[str | dict[str, str]] | None
|
|
73
|
+
filter_expr: str | dict[str, Any] | None
|
|
74
|
+
ids: list[str] | None
|
|
75
|
+
filter_fields: set[str]
|
|
76
|
+
dates: list[str]
|
|
77
|
+
chunk_bbox_4326: list[float]
|
|
78
|
+
selected_bands: list[str]
|
|
79
|
+
chunk_affine: Affine
|
|
80
|
+
dst_crs: CRS
|
|
81
|
+
chunk_width: int
|
|
82
|
+
chunk_height: int
|
|
83
|
+
nodata: float | None
|
|
84
|
+
mosaic_method_cls: type[MosaicMethodBase] | None
|
|
85
|
+
store: Any | None
|
|
86
|
+
max_concurrent_reads: int
|
|
87
|
+
warp_cache: dict
|
|
88
|
+
path_fn: Callable[[str], str] | None
|
|
89
|
+
|
|
90
|
+
|
|
91
|
+
def _get_or_create_background_loop() -> asyncio.AbstractEventLoop:
|
|
92
|
+
"""Return the persistent background event loop for the current thread.
|
|
93
|
+
|
|
94
|
+
Creates the loop, its bounded reprojection executor, and its daemon runner
|
|
95
|
+
thread on first call from a given thread. Subsequent calls on the same
|
|
96
|
+
thread return the cached loop immediately.
|
|
97
|
+
|
|
98
|
+
The loop is stored on ``threading.local`` so each thread (each dask worker,
|
|
99
|
+
each Jupyter kernel callback thread) has its own independent loop and
|
|
100
|
+
executor — tasks on different threads do not share a pool.
|
|
101
|
+
|
|
102
|
+
Using a persistent loop (rather than a fresh one per call) ensures that
|
|
103
|
+
any in-flight callbacks from ``async_geotiff``/``obstore`` background
|
|
104
|
+
threads can always be delivered, avoiding ``RuntimeError: Event loop is
|
|
105
|
+
closed`` errors from callbacks that fire after a fresh loop is torn down.
|
|
106
|
+
"""
|
|
107
|
+
loop: asyncio.AbstractEventLoop | None = getattr(_tls, "loop", None)
|
|
108
|
+
if loop is not None and loop.is_running():
|
|
109
|
+
return loop
|
|
110
|
+
|
|
111
|
+
loop = asyncio.new_event_loop()
|
|
112
|
+
loop.set_default_executor(
|
|
113
|
+
concurrent.futures.ThreadPoolExecutor(
|
|
114
|
+
max_workers=get_max_workers(),
|
|
115
|
+
thread_name_prefix="lazycogs-reproject",
|
|
116
|
+
)
|
|
117
|
+
)
|
|
118
|
+
t = threading.Thread(target=loop.run_forever, daemon=True, name="lazycogs-loop")
|
|
119
|
+
t.start()
|
|
120
|
+
_tls.loop = loop
|
|
121
|
+
return loop
|
|
122
|
+
|
|
123
|
+
|
|
124
|
+
def _run_coroutine(coro: Any) -> Any:
|
|
125
|
+
"""Run an async coroutine from sync code.
|
|
126
|
+
|
|
127
|
+
Submits the coroutine to a persistent per-thread background event loop,
|
|
128
|
+
blocking until it completes. The background loop is created on the first
|
|
129
|
+
call from a given thread (dask worker, Jupyter kernel thread, etc.) and
|
|
130
|
+
reused for all subsequent calls on that same thread.
|
|
131
|
+
|
|
132
|
+
Args:
|
|
133
|
+
coro: The coroutine to execute.
|
|
134
|
+
|
|
135
|
+
Returns:
|
|
136
|
+
The return value of the coroutine.
|
|
137
|
+
|
|
138
|
+
"""
|
|
139
|
+
loop = _get_or_create_background_loop()
|
|
140
|
+
return asyncio.run_coroutine_threadsafe(coro, loop).result()
|
|
141
|
+
|
|
142
|
+
|
|
143
|
+
@dataclass
|
|
144
|
+
class _SpatialWindow:
|
|
145
|
+
"""Resolved spatial indexing window.
|
|
146
|
+
|
|
147
|
+
Attributes:
|
|
148
|
+
chunk_affine: Affine transform of the chunk (top-left origin).
|
|
149
|
+
chunk_bbox_4326: ``[minx, miny, maxx, maxy]`` in EPSG:4326.
|
|
150
|
+
chunk_height: Chunk height in pixels.
|
|
151
|
+
chunk_width: Chunk width in pixels.
|
|
152
|
+
x_start: First x pixel in the destination grid.
|
|
153
|
+
squeeze_y: Whether the y dimension should be squeezed on return.
|
|
154
|
+
squeeze_x: Whether the x dimension should be squeezed on return.
|
|
155
|
+
|
|
156
|
+
"""
|
|
157
|
+
|
|
158
|
+
chunk_affine: Affine
|
|
159
|
+
chunk_bbox_4326: list[float]
|
|
160
|
+
chunk_height: int
|
|
161
|
+
chunk_width: int
|
|
162
|
+
x_start: int
|
|
163
|
+
squeeze_y: bool
|
|
164
|
+
squeeze_x: bool
|
|
165
|
+
|
|
166
|
+
|
|
167
|
+
def _resolve_time_indices(
|
|
168
|
+
time_key: int | np.integer | slice, n_dates: int
|
|
169
|
+
) -> tuple[list[int], bool]:
|
|
170
|
+
"""Resolve a time indexer to a list of integer indices.
|
|
171
|
+
|
|
172
|
+
Args:
|
|
173
|
+
time_key: Integer or slice indexer for the time dimension.
|
|
174
|
+
n_dates: Total number of dates (size of the time dimension).
|
|
175
|
+
|
|
176
|
+
Returns:
|
|
177
|
+
``(time_indices, squeeze_time)`` where ``squeeze_time`` is ``True``
|
|
178
|
+
when ``time_key`` was a scalar integer.
|
|
179
|
+
|
|
180
|
+
"""
|
|
181
|
+
if isinstance(time_key, (int, np.integer)):
|
|
182
|
+
return [int(time_key)], True
|
|
183
|
+
start = time_key.start if time_key.start is not None else 0
|
|
184
|
+
stop = time_key.stop if time_key.stop is not None else n_dates
|
|
185
|
+
step = time_key.step if time_key.step is not None else 1
|
|
186
|
+
return list(range(start, stop, step)), False
|
|
187
|
+
|
|
188
|
+
|
|
189
|
+
def _resolve_spatial_window(
|
|
190
|
+
y_key: int | np.integer | slice,
|
|
191
|
+
x_key: int | np.integer | slice,
|
|
192
|
+
dst_height: int,
|
|
193
|
+
dst_width: int,
|
|
194
|
+
dst_affine: Affine,
|
|
195
|
+
dst_crs: CRS,
|
|
196
|
+
dst_to_4326: Transformer | None,
|
|
197
|
+
) -> _SpatialWindow:
|
|
198
|
+
"""Resolve spatial indexers to a concrete chunk window.
|
|
199
|
+
|
|
200
|
+
Converts logical ascending y indices (south-to-north, as exposed to
|
|
201
|
+
xarray) to physical top-down row offsets (north-to-south, as stored in
|
|
202
|
+
the COG), then computes the chunk affine transform and EPSG:4326 bounding
|
|
203
|
+
box.
|
|
204
|
+
|
|
205
|
+
Args:
|
|
206
|
+
y_key: Integer or slice indexer for the y dimension.
|
|
207
|
+
x_key: Integer or slice indexer for the x dimension.
|
|
208
|
+
dst_height: Full output grid height in pixels.
|
|
209
|
+
dst_width: Full output grid width in pixels.
|
|
210
|
+
dst_affine: Affine transform of the full output grid.
|
|
211
|
+
dst_crs: CRS of the output grid.
|
|
212
|
+
dst_to_4326: Pre-built transformer from ``dst_crs`` to EPSG:4326, or
|
|
213
|
+
``None`` when ``dst_crs`` is already EPSG:4326.
|
|
214
|
+
|
|
215
|
+
Returns:
|
|
216
|
+
A :class:`_SpatialWindow` describing the chunk geometry.
|
|
217
|
+
|
|
218
|
+
"""
|
|
219
|
+
if isinstance(y_key, (int, np.integer)):
|
|
220
|
+
yi = int(y_key)
|
|
221
|
+
y_key = slice(yi, yi + 1)
|
|
222
|
+
squeeze_y = True
|
|
223
|
+
else:
|
|
224
|
+
squeeze_y = False
|
|
225
|
+
|
|
226
|
+
if isinstance(x_key, (int, np.integer)):
|
|
227
|
+
xi = int(x_key)
|
|
228
|
+
x_key = slice(xi, xi + 1)
|
|
229
|
+
squeeze_x = True
|
|
230
|
+
else:
|
|
231
|
+
squeeze_x = False
|
|
232
|
+
|
|
233
|
+
y_start_logical = y_key.start if y_key.start is not None else 0
|
|
234
|
+
y_stop_logical = y_key.stop if y_key.stop is not None else dst_height
|
|
235
|
+
x_start = x_key.start if x_key.start is not None else 0
|
|
236
|
+
x_stop = x_key.stop if x_key.stop is not None else dst_width
|
|
237
|
+
|
|
238
|
+
# logical index 0 = southernmost = physical row (dst_height - 1)
|
|
239
|
+
y_start_physical = dst_height - y_stop_logical
|
|
240
|
+
y_stop_physical = dst_height - y_start_logical
|
|
241
|
+
|
|
242
|
+
chunk_height = y_stop_physical - y_start_physical
|
|
243
|
+
chunk_width = x_stop - x_start
|
|
244
|
+
|
|
245
|
+
chunk_affine = dst_affine * Affine.translation(x_start, y_start_physical)
|
|
246
|
+
|
|
247
|
+
minx = chunk_affine.c
|
|
248
|
+
maxy = chunk_affine.f
|
|
249
|
+
maxx = minx + chunk_width * chunk_affine.a
|
|
250
|
+
miny = maxy + chunk_height * chunk_affine.e # e < 0
|
|
251
|
+
|
|
252
|
+
if dst_to_4326 is None:
|
|
253
|
+
chunk_bbox_4326 = [minx, miny, maxx, maxy]
|
|
254
|
+
else:
|
|
255
|
+
xs, ys = dst_to_4326.transform(
|
|
256
|
+
[minx, maxx, minx, maxx],
|
|
257
|
+
[maxy, maxy, miny, miny],
|
|
258
|
+
)
|
|
259
|
+
chunk_bbox_4326 = [
|
|
260
|
+
float(min(xs)),
|
|
261
|
+
float(min(ys)),
|
|
262
|
+
float(max(xs)),
|
|
263
|
+
float(max(ys)),
|
|
264
|
+
]
|
|
265
|
+
|
|
266
|
+
return _SpatialWindow(
|
|
267
|
+
chunk_affine=chunk_affine,
|
|
268
|
+
chunk_bbox_4326=chunk_bbox_4326,
|
|
269
|
+
chunk_height=chunk_height,
|
|
270
|
+
chunk_width=chunk_width,
|
|
271
|
+
x_start=x_start,
|
|
272
|
+
squeeze_y=squeeze_y,
|
|
273
|
+
squeeze_x=squeeze_x,
|
|
274
|
+
)
|
|
275
|
+
|
|
276
|
+
|
|
277
|
+
def _search_items(
|
|
278
|
+
client: DuckdbClient,
|
|
279
|
+
parquet_path: str,
|
|
280
|
+
bbox: list[float],
|
|
281
|
+
date: str,
|
|
282
|
+
sortby: str | list[str | dict[str, str]] | None,
|
|
283
|
+
filter_expr: str | dict[str, Any] | None,
|
|
284
|
+
ids: list[str] | None,
|
|
285
|
+
filter_fields: set[str],
|
|
286
|
+
label: str,
|
|
287
|
+
) -> list[Any]:
|
|
288
|
+
"""Query the STAC parquet for items overlapping a chunk.
|
|
289
|
+
|
|
290
|
+
Args:
|
|
291
|
+
client: ``DuckdbClient`` instance.
|
|
292
|
+
parquet_path: Path to the geoparquet file or hive-partitioned directory.
|
|
293
|
+
bbox: ``[minx, miny, maxx, maxy]`` bounding box in EPSG:4326.
|
|
294
|
+
date: Acquisition date string (``"YYYY-MM-DD"``).
|
|
295
|
+
sortby: Optional sort keys forwarded to ``client.search``.
|
|
296
|
+
filter_expr: Optional CQL2 filter forwarded to ``client.search``.
|
|
297
|
+
ids: Optional STAC item IDs forwarded to ``client.search``.
|
|
298
|
+
filter_fields: Field names extracted from ``filter_expr`` (added to
|
|
299
|
+
the ``include`` list so DuckDB returns them).
|
|
300
|
+
label: Short identifier used in debug log messages (e.g. the band name
|
|
301
|
+
or a list of band names).
|
|
302
|
+
|
|
303
|
+
Returns:
|
|
304
|
+
List of STAC items returned by DuckDB.
|
|
305
|
+
|
|
306
|
+
"""
|
|
307
|
+
t0 = time.perf_counter()
|
|
308
|
+
items = client.search(
|
|
309
|
+
parquet_path,
|
|
310
|
+
bbox=bbox,
|
|
311
|
+
datetime=date,
|
|
312
|
+
sortby=sortby,
|
|
313
|
+
filter=filter_expr,
|
|
314
|
+
ids=ids,
|
|
315
|
+
include=list(
|
|
316
|
+
{"id", "assets"}.union(filter_fields).union(_sortby_fields(sortby))
|
|
317
|
+
),
|
|
318
|
+
)
|
|
319
|
+
logger.debug(
|
|
320
|
+
"duckdb_client.search %s date=%s returned %d items in %.3fs",
|
|
321
|
+
label,
|
|
322
|
+
date,
|
|
323
|
+
len(items),
|
|
324
|
+
time.perf_counter() - t0,
|
|
325
|
+
)
|
|
326
|
+
if items and logger.isEnabledFor(logging.DEBUG):
|
|
327
|
+
unexpected = {k for k in items[0] if k not in {"id", "assets"}}
|
|
328
|
+
if unexpected:
|
|
329
|
+
logger.debug(
|
|
330
|
+
"duckdb_client.search returned unexpected fields %s — "
|
|
331
|
+
"include filter may not be respected",
|
|
332
|
+
unexpected,
|
|
333
|
+
)
|
|
334
|
+
return items
|
|
335
|
+
|
|
336
|
+
|
|
337
|
+
async def _run_one_date(
|
|
338
|
+
t_idx: int,
|
|
339
|
+
plan: _ChunkReadPlan,
|
|
340
|
+
) -> dict[str, np.ndarray] | None:
|
|
341
|
+
"""Read and mosaic all COGs for a single time step.
|
|
342
|
+
|
|
343
|
+
Issues one DuckDB query for items overlapping the chunk at this date, then
|
|
344
|
+
calls async_mosaic_chunk to fetch and reproject all tiles.
|
|
345
|
+
Returns None if no items match the query.
|
|
346
|
+
|
|
347
|
+
Args:
|
|
348
|
+
t_idx: Index into ``plan.dates`` for the time step to read.
|
|
349
|
+
plan: Read plan carrying all parameters for this chunk.
|
|
350
|
+
|
|
351
|
+
Returns:
|
|
352
|
+
Per-band arrays keyed by band name, or ``None`` if no items matched.
|
|
353
|
+
|
|
354
|
+
"""
|
|
355
|
+
date = plan.dates[t_idx]
|
|
356
|
+
|
|
357
|
+
with plan.duckdb_lock:
|
|
358
|
+
items = _search_items(
|
|
359
|
+
plan.duckdb_client,
|
|
360
|
+
plan.parquet_path,
|
|
361
|
+
plan.chunk_bbox_4326,
|
|
362
|
+
date,
|
|
363
|
+
plan.sortby,
|
|
364
|
+
plan.filter_expr,
|
|
365
|
+
plan.ids,
|
|
366
|
+
plan.filter_fields,
|
|
367
|
+
label=f"bands={plan.selected_bands!r}",
|
|
368
|
+
)
|
|
369
|
+
|
|
370
|
+
if not items:
|
|
371
|
+
return None
|
|
372
|
+
t0 = time.perf_counter()
|
|
373
|
+
chunk_result = await async_mosaic_chunk(
|
|
374
|
+
items=items,
|
|
375
|
+
bands=plan.selected_bands,
|
|
376
|
+
chunk_affine=plan.chunk_affine,
|
|
377
|
+
dst_crs=plan.dst_crs,
|
|
378
|
+
chunk_width=plan.chunk_width,
|
|
379
|
+
chunk_height=plan.chunk_height,
|
|
380
|
+
nodata=plan.nodata,
|
|
381
|
+
mosaic_method_cls=plan.mosaic_method_cls,
|
|
382
|
+
store=plan.store,
|
|
383
|
+
max_concurrent_reads=plan.max_concurrent_reads,
|
|
384
|
+
warp_cache=plan.warp_cache,
|
|
385
|
+
path_fn=plan.path_fn,
|
|
386
|
+
)
|
|
387
|
+
logger.debug(
|
|
388
|
+
"async_mosaic_chunk bands=%r date=%s (%d items, %dx%d px) took %.3fs",
|
|
389
|
+
plan.selected_bands,
|
|
390
|
+
date,
|
|
391
|
+
len(items),
|
|
392
|
+
plan.chunk_width,
|
|
393
|
+
plan.chunk_height,
|
|
394
|
+
time.perf_counter() - t0,
|
|
395
|
+
)
|
|
396
|
+
return chunk_result
|
|
397
|
+
|
|
398
|
+
|
|
399
|
+
async def _run_mosaic_all_dates(
|
|
400
|
+
time_indices: list[int],
|
|
401
|
+
plan: _ChunkReadPlan,
|
|
402
|
+
) -> list[dict[str, np.ndarray] | None]:
|
|
403
|
+
"""Run all time steps concurrently inside a single event loop.
|
|
404
|
+
|
|
405
|
+
DuckDB queries are issued sequentially (the threading.Lock already
|
|
406
|
+
serialises access). Mosaic coroutines for all time steps are gathered
|
|
407
|
+
concurrently so COG reads and reprojections overlap across time steps.
|
|
408
|
+
|
|
409
|
+
Args:
|
|
410
|
+
time_indices: Ordered list of time-dimension indices to materialise.
|
|
411
|
+
plan: Read plan carrying all parameters for this chunk.
|
|
412
|
+
|
|
413
|
+
Returns:
|
|
414
|
+
One entry per time index; ``None`` where no items matched.
|
|
415
|
+
|
|
416
|
+
"""
|
|
417
|
+
return list(await asyncio.gather(*[_run_one_date(t, plan) for t in time_indices]))
|
|
418
|
+
|
|
419
|
+
|
|
420
|
+
@dataclass
|
|
421
|
+
class MultiBandStacBackendArray(BackendArray):
|
|
422
|
+
"""Lazy ``(band, time, y, x)`` array for a STAC collection.
|
|
423
|
+
|
|
424
|
+
One instance is created at ``open()`` time. No pixel I/O happens until
|
|
425
|
+
``__getitem__`` is called inside a dask task. Reads all selected bands
|
|
426
|
+
together per time step via
|
|
427
|
+
:func:`~lazycogs._chunk_reader.async_mosaic_chunk`, issuing a
|
|
428
|
+
single DuckDB query per time step and sharing reprojection warp maps across
|
|
429
|
+
bands that have identical source geometry.
|
|
430
|
+
|
|
431
|
+
Attributes:
|
|
432
|
+
parquet_path: Path to the geoparquet file or hive-partitioned directory
|
|
433
|
+
passed to ``duckdb_client.search``.
|
|
434
|
+
duckdb_client: ``DuckdbClient`` instance used for all STAC queries.
|
|
435
|
+
Constructed with default settings in :func:`open` when not supplied
|
|
436
|
+
by the caller.
|
|
437
|
+
bands: Ordered list of STAC asset keys, one per band.
|
|
438
|
+
dates: Sorted list of unique acquisition date strings
|
|
439
|
+
(``"YYYY-MM-DD"``), one entry per time step.
|
|
440
|
+
dst_affine: Affine transform of the full output grid.
|
|
441
|
+
dst_crs: CRS of the output grid.
|
|
442
|
+
bbox_4326: ``[minx, miny, maxx, maxy]`` in EPSG:4326, used as the
|
|
443
|
+
coarse spatial filter for the initial parquet query.
|
|
444
|
+
sortby: Optional list of ``rustac`` sort keys passed to DuckDB
|
|
445
|
+
queries (e.g. ``["-properties.datetime"]``).
|
|
446
|
+
filter: CQL2 filter expression (text string or JSON dict) forwarded
|
|
447
|
+
to per-chunk DuckDB queries.
|
|
448
|
+
ids: STAC item IDs forwarded to per-chunk DuckDB queries.
|
|
449
|
+
dst_width: Full output grid width in pixels.
|
|
450
|
+
dst_height: Full output grid height in pixels.
|
|
451
|
+
dtype: NumPy dtype of the output array.
|
|
452
|
+
nodata: No-data fill value, or ``None``.
|
|
453
|
+
mosaic_method_cls: Mosaic method class instantiated per chunk, or
|
|
454
|
+
``None`` to use the default
|
|
455
|
+
:class:`~lazycogs._mosaic_methods.FirstMethod`.
|
|
456
|
+
store: Pre-configured obstore ``ObjectStore`` instance shared across
|
|
457
|
+
all chunk reads. When ``None``, each asset HREF is resolved to a
|
|
458
|
+
store via the thread-local cache in
|
|
459
|
+
:func:`~lazycogs._store.resolve`.
|
|
460
|
+
max_concurrent_reads: Maximum number of COG reads to run concurrently
|
|
461
|
+
per chunk. Limits peak in-flight memory when a chunk overlaps
|
|
462
|
+
many items. Defaults to 32.
|
|
463
|
+
path_from_href: Optional callable ``(href: str) -> str`` that extracts
|
|
464
|
+
the object path from an asset HREF. When provided, it replaces the
|
|
465
|
+
default ``urlparse``-based extraction in
|
|
466
|
+
:func:`~lazycogs._store.resolve`. Most useful when combined with
|
|
467
|
+
a custom ``store`` whose root does not align with the URL structure
|
|
468
|
+
of the asset HREFs (e.g. Azure Blob Storage with a container-rooted
|
|
469
|
+
store).
|
|
470
|
+
shape: ``(n_bands, n_dates, dst_height, dst_width)``. Derived from
|
|
471
|
+
the other fields; not accepted as a constructor argument.
|
|
472
|
+
|
|
473
|
+
"""
|
|
474
|
+
|
|
475
|
+
parquet_path: str
|
|
476
|
+
duckdb_client: DuckdbClient
|
|
477
|
+
bands: list[str]
|
|
478
|
+
dates: list[str]
|
|
479
|
+
dst_affine: Affine
|
|
480
|
+
dst_crs: CRS
|
|
481
|
+
bbox_4326: list[float]
|
|
482
|
+
sortby: str | list[str | dict[str, str]] | None
|
|
483
|
+
filter: str | dict[str, Any] | None
|
|
484
|
+
ids: list[str] | None
|
|
485
|
+
dst_width: int
|
|
486
|
+
dst_height: int
|
|
487
|
+
dtype: np.dtype
|
|
488
|
+
nodata: float | None
|
|
489
|
+
mosaic_method_cls: type[MosaicMethodBase] | None = field(default=None)
|
|
490
|
+
store: Any | None = field(default=None)
|
|
491
|
+
max_concurrent_reads: int = field(default=32)
|
|
492
|
+
path_from_href: Callable[[str], str] | None = field(default=None)
|
|
493
|
+
shape: tuple[int, ...] = field(init=False)
|
|
494
|
+
_dst_to_4326: Transformer | None = field(init=False, repr=False, compare=False)
|
|
495
|
+
_duckdb_lock: threading.Lock = field(
|
|
496
|
+
init=False, repr=False, compare=False, default_factory=threading.Lock
|
|
497
|
+
)
|
|
498
|
+
|
|
499
|
+
def __post_init__(self) -> None:
|
|
500
|
+
"""Derive shape and cache the dst→EPSG:4326 transformer."""
|
|
501
|
+
self.shape = (len(self.bands), len(self.dates), self.dst_height, self.dst_width)
|
|
502
|
+
epsg_4326 = CRS.from_epsg(4326)
|
|
503
|
+
if self.dst_crs.equals(epsg_4326):
|
|
504
|
+
self._dst_to_4326: Transformer | None = None
|
|
505
|
+
else:
|
|
506
|
+
self._dst_to_4326 = Transformer.from_crs(
|
|
507
|
+
self.dst_crs, epsg_4326, always_xy=True
|
|
508
|
+
)
|
|
509
|
+
|
|
510
|
+
def __repr__(self) -> str:
|
|
511
|
+
"""Return a compact string representation."""
|
|
512
|
+
return f"MultiBandStacBackendArray(bands={self.bands!r}, shape={self.shape})"
|
|
513
|
+
|
|
514
|
+
def __getitem__(self, key: indexing.ExplicitIndexer) -> np.ndarray:
|
|
515
|
+
"""Return the data for the requested index.
|
|
516
|
+
|
|
517
|
+
Args:
|
|
518
|
+
key: An xarray ``ExplicitIndexer``.
|
|
519
|
+
|
|
520
|
+
Returns:
|
|
521
|
+
A numpy array with shape determined by the indexing key.
|
|
522
|
+
|
|
523
|
+
"""
|
|
524
|
+
return indexing.explicit_indexing_adapter(
|
|
525
|
+
key,
|
|
526
|
+
self.shape,
|
|
527
|
+
indexing.IndexingSupport.BASIC,
|
|
528
|
+
self._raw_getitem,
|
|
529
|
+
)
|
|
530
|
+
|
|
531
|
+
def _raw_getitem(self, key: tuple[Any, ...]) -> np.ndarray:
|
|
532
|
+
"""Materialise the chunk identified by ``key``.
|
|
533
|
+
|
|
534
|
+
Reads all selected bands together per time step via
|
|
535
|
+
:func:`~lazycogs._chunk_reader.async_mosaic_chunk`, issuing
|
|
536
|
+
a single DuckDB query per time step and sharing reprojection warp maps
|
|
537
|
+
across bands that have identical source geometry.
|
|
538
|
+
|
|
539
|
+
Args:
|
|
540
|
+
key: A tuple of ``int | slice`` objects for the
|
|
541
|
+
``(band, time, y, x)`` dimensions.
|
|
542
|
+
|
|
543
|
+
Returns:
|
|
544
|
+
Numpy array with shape determined by the indexing key.
|
|
545
|
+
|
|
546
|
+
"""
|
|
547
|
+
band_key, time_key, y_key, x_key = key
|
|
548
|
+
|
|
549
|
+
# -- Band dimension --------------------------------------------------
|
|
550
|
+
n_bands = len(self.bands)
|
|
551
|
+
if isinstance(band_key, (int, np.integer)):
|
|
552
|
+
band_indices: list[int] = [int(band_key)]
|
|
553
|
+
squeeze_band = True
|
|
554
|
+
else:
|
|
555
|
+
start = band_key.start if band_key.start is not None else 0
|
|
556
|
+
stop = band_key.stop if band_key.stop is not None else n_bands
|
|
557
|
+
step = band_key.step if band_key.step is not None else 1
|
|
558
|
+
band_indices = list(range(start, stop, step))
|
|
559
|
+
squeeze_band = False
|
|
560
|
+
|
|
561
|
+
selected_bands = [self.bands[b] for b in band_indices]
|
|
562
|
+
|
|
563
|
+
time_indices, squeeze_time = _resolve_time_indices(time_key, len(self.dates))
|
|
564
|
+
win = _resolve_spatial_window(
|
|
565
|
+
y_key,
|
|
566
|
+
x_key,
|
|
567
|
+
self.dst_height,
|
|
568
|
+
self.dst_width,
|
|
569
|
+
self.dst_affine,
|
|
570
|
+
self.dst_crs,
|
|
571
|
+
self._dst_to_4326,
|
|
572
|
+
)
|
|
573
|
+
|
|
574
|
+
fill = self.nodata if self.nodata is not None else 0
|
|
575
|
+
|
|
576
|
+
filter_fields = _extract_filter_fields(self.filter) if self.filter else set()
|
|
577
|
+
plan = _ChunkReadPlan(
|
|
578
|
+
duckdb_client=self.duckdb_client,
|
|
579
|
+
parquet_path=self.parquet_path,
|
|
580
|
+
duckdb_lock=self._duckdb_lock,
|
|
581
|
+
sortby=self.sortby,
|
|
582
|
+
filter_expr=self.filter,
|
|
583
|
+
ids=self.ids,
|
|
584
|
+
filter_fields=filter_fields,
|
|
585
|
+
dates=self.dates,
|
|
586
|
+
chunk_bbox_4326=win.chunk_bbox_4326,
|
|
587
|
+
selected_bands=selected_bands,
|
|
588
|
+
chunk_affine=win.chunk_affine,
|
|
589
|
+
dst_crs=self.dst_crs,
|
|
590
|
+
chunk_width=win.chunk_width,
|
|
591
|
+
chunk_height=win.chunk_height,
|
|
592
|
+
nodata=self.nodata,
|
|
593
|
+
mosaic_method_cls=self.mosaic_method_cls,
|
|
594
|
+
store=self.store,
|
|
595
|
+
max_concurrent_reads=self.max_concurrent_reads,
|
|
596
|
+
warp_cache={},
|
|
597
|
+
path_fn=self.path_from_href,
|
|
598
|
+
)
|
|
599
|
+
|
|
600
|
+
all_chunk_data = _run_coroutine(_run_mosaic_all_dates(time_indices, plan))
|
|
601
|
+
|
|
602
|
+
out_shape = (
|
|
603
|
+
len(band_indices),
|
|
604
|
+
len(time_indices),
|
|
605
|
+
win.chunk_height,
|
|
606
|
+
win.chunk_width,
|
|
607
|
+
)
|
|
608
|
+
result: np.ndarray | None = None
|
|
609
|
+
|
|
610
|
+
for i, chunk_data in enumerate(all_chunk_data):
|
|
611
|
+
if chunk_data is None:
|
|
612
|
+
continue
|
|
613
|
+
if result is None:
|
|
614
|
+
result = np.full(out_shape, fill, dtype=self.dtype)
|
|
615
|
+
for bi, band in enumerate(selected_bands):
|
|
616
|
+
arr = chunk_data[band]
|
|
617
|
+
slice_ = arr[0] if arr.ndim == 3 else arr
|
|
618
|
+
result[bi, i] = slice_.astype(self.dtype, copy=False)
|
|
619
|
+
|
|
620
|
+
if result is None:
|
|
621
|
+
result = np.full(out_shape, fill, dtype=self.dtype)
|
|
622
|
+
|
|
623
|
+
# Physical data is top-down; flip to ascending y order for xarray.
|
|
624
|
+
result = result[:, :, ::-1, :]
|
|
625
|
+
|
|
626
|
+
# result shape: (n_selected_bands, n_time, chunk_height, chunk_width)
|
|
627
|
+
# Apply squeezes in axis order: time (axis 1), band (axis 0), y (-2), x (-1).
|
|
628
|
+
out: np.ndarray = result # type: ignore[assignment]
|
|
629
|
+
if squeeze_time:
|
|
630
|
+
out = out[:, 0, :, :]
|
|
631
|
+
if squeeze_band:
|
|
632
|
+
out = out[0]
|
|
633
|
+
if win.squeeze_y:
|
|
634
|
+
out = np.take(out, 0, axis=-2)
|
|
635
|
+
if win.squeeze_x:
|
|
636
|
+
out = out[..., 0]
|
|
637
|
+
return out
|