lazycogs 0.1.2__tar.gz → 0.2.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.3
2
2
  Name: lazycogs
3
- Version: 0.1.2
3
+ Version: 0.2.0
4
4
  Summary: Lazily load COG assets from STAC items into xarray DataArrays using async-geotiff
5
5
  Keywords: COG,STAC,xarray,geotiff,raster,geospatial
6
6
  Author: Henry Rodman
@@ -58,7 +58,7 @@ Description-Content-Type: text/markdown
58
58
  [![Python Versions](https://img.shields.io/pypi/pyversions/lazycogs)](https://pypi.org/project/lazycogs/)
59
59
  [![License](https://img.shields.io/github/license/developmentseed/lazycogs)](https://github.com/developmentseed/lazycogs/blob/main/LICENSE)
60
60
 
61
- Open a lazy `(time, band, y, x)` xarray DataArray from thousands of cloud-optimized geotiffs (COGs). No GDAL required.
61
+ Open a lazy `(band, time, y, x)` xarray DataArray from thousands of cloud-optimized geotiffs (COGs). No GDAL required.
62
62
 
63
63
  ## What is lazycogs?
64
64
 
@@ -98,7 +98,7 @@ dst_crs = "EPSG:32615"
98
98
  dst_bbox = (380000.0, 4928000.0, 420000.0, 4984000.0)
99
99
 
100
100
  # transform to 4326 for STAC search
101
- transformer = Transformer.from_crs(dst_crs, "epsg:4326", alwaysxy=True)
101
+ transformer = Transformer.from_crs(dst_crs, "epsg:4326", always_xy=True)
102
102
  bbox_4326 = transformer.transform_bounds(*dst_bbox)
103
103
 
104
104
  # Search a STAC API and cache results to a local stac-geoparquet file.
@@ -110,7 +110,7 @@ await rustac.search_to(
110
110
  bbox=bbox_4326,
111
111
  )
112
112
 
113
- # Open a fully lazy (time, band, y, x) DataArray. No COGs are read yet.
113
+ # Open a fully lazy (band, time, y, x) DataArray. No COGs are read yet.
114
114
  da = lazycogs.open(
115
115
  "items.parquet",
116
116
  bbox=dst_bbox,
@@ -5,7 +5,7 @@
5
5
  [![Python Versions](https://img.shields.io/pypi/pyversions/lazycogs)](https://pypi.org/project/lazycogs/)
6
6
  [![License](https://img.shields.io/github/license/developmentseed/lazycogs)](https://github.com/developmentseed/lazycogs/blob/main/LICENSE)
7
7
 
8
- Open a lazy `(time, band, y, x)` xarray DataArray from thousands of cloud-optimized geotiffs (COGs). No GDAL required.
8
+ Open a lazy `(band, time, y, x)` xarray DataArray from thousands of cloud-optimized geotiffs (COGs). No GDAL required.
9
9
 
10
10
  ## What is lazycogs?
11
11
 
@@ -45,7 +45,7 @@ dst_crs = "EPSG:32615"
45
45
  dst_bbox = (380000.0, 4928000.0, 420000.0, 4984000.0)
46
46
 
47
47
  # transform to 4326 for STAC search
48
- transformer = Transformer.from_crs(dst_crs, "epsg:4326", alwaysxy=True)
48
+ transformer = Transformer.from_crs(dst_crs, "epsg:4326", always_xy=True)
49
49
  bbox_4326 = transformer.transform_bounds(*dst_bbox)
50
50
 
51
51
  # Search a STAC API and cache results to a local stac-geoparquet file.
@@ -57,7 +57,7 @@ await rustac.search_to(
57
57
  bbox=bbox_4326,
58
58
  )
59
59
 
60
- # Open a fully lazy (time, band, y, x) DataArray. No COGs are read yet.
60
+ # Open a fully lazy (band, time, y, x) DataArray. No COGs are read yet.
61
61
  da = lazycogs.open(
62
62
  "items.parquet",
63
63
  bbox=dst_bbox,
@@ -1,6 +1,6 @@
1
1
  [project]
2
2
  name = "lazycogs"
3
- version = "0.1.2"
3
+ version = "0.2.0"
4
4
  description = "Lazily load COG assets from STAC items into xarray DataArrays using async-geotiff"
5
5
  readme = "README.md"
6
6
  license = {file = "LICENSE"}
@@ -1,5 +1,6 @@
1
1
  """lazycogs: lazy xarray DataArrays from STAC COG collections."""
2
2
 
3
+ from lazycogs._chunk_reader import read_chunk, read_chunk_async
3
4
  from lazycogs._core import open # noqa: A004
4
5
  from lazycogs._executor import set_reproject_workers
5
6
  from lazycogs._explain import ( # noqa: F401 — registers da.stac_cog accessor
@@ -35,6 +36,8 @@ __all__ = [
35
36
  "StdevMethod",
36
37
  "align_bbox",
37
38
  "open",
39
+ "read_chunk",
40
+ "read_chunk_async",
38
41
  "set_reproject_workers",
39
42
  "store_for",
40
43
  ]
@@ -3,9 +3,7 @@
3
3
  from __future__ import annotations
4
4
 
5
5
  import asyncio
6
- import concurrent.futures
7
6
  import logging
8
- import threading
9
7
  import time
10
8
  from dataclasses import dataclass, field
11
9
  from typing import TYPE_CHECKING, Any
@@ -16,19 +14,18 @@ from pyproj import CRS, Transformer
16
14
  from xarray.backends.common import BackendArray
17
15
  from xarray.core import indexing
18
16
 
19
- from lazycogs._chunk_reader import async_mosaic_chunk
17
+ from lazycogs._chunk_reader import read_chunk_async
20
18
  from lazycogs._cql2 import _extract_filter_fields, _sortby_fields
21
- from lazycogs._executor import get_max_workers
19
+ from lazycogs._executor import (
20
+ _DUCKDB_EXECUTOR,
21
+ _run_coroutine,
22
+ )
22
23
 
23
24
  logger = logging.getLogger(__name__)
24
25
 
25
26
 
26
- _tls = threading.local()
27
-
28
27
  if TYPE_CHECKING:
29
- from asyncio.futures import Future
30
28
  from collections.abc import Callable
31
- from types import CoroutineType
32
29
 
33
30
  from rustac import DuckdbClient
34
31
 
@@ -39,8 +36,8 @@ if TYPE_CHECKING:
39
36
  class _ChunkReadPlan:
40
37
  """Everything needed to materialise one chunk across all its time steps.
41
38
 
42
- Built once in ``_raw_getitem`` and passed through to
43
- ``_run_mosaic_all_dates`` and ``_run_one_date``. Frozen to make the
39
+ Built once in ``_async_getitem`` and passed through to
40
+ ``_read_chunk_all_dates`` and ``_run_one_date``. Frozen to make the
44
41
  read-only intent explicit.
45
42
 
46
43
  Note: ``warp_cache`` is a mutable dict despite the frozen dataclass. This
@@ -51,7 +48,6 @@ class _ChunkReadPlan:
51
48
  Attributes:
52
49
  duckdb_client: ``DuckdbClient`` instance used for STAC queries.
53
50
  parquet_path: Path to the geoparquet file or hive-partitioned directory.
54
- duckdb_lock: Lock serialising access to ``duckdb_client``.
55
51
  sortby: Optional sort keys forwarded to ``client.search``.
56
52
  filter_expr: Optional CQL2 filter forwarded to ``client.search``.
57
53
  ids: Optional STAC item IDs forwarded to ``client.search``.
@@ -74,7 +70,6 @@ class _ChunkReadPlan:
74
70
 
75
71
  duckdb_client: DuckdbClient
76
72
  parquet_path: str
77
- duckdb_lock: threading.Lock
78
73
  sortby: str | list[str | dict[str, str]] | None
79
74
  filter_expr: str | dict[str, Any] | None
80
75
  ids: list[str] | None
@@ -94,58 +89,6 @@ class _ChunkReadPlan:
94
89
  path_fn: Callable[[str], str] | None
95
90
 
96
91
 
97
- def _get_or_create_background_loop() -> asyncio.AbstractEventLoop:
98
- """Return the persistent background event loop for the current thread.
99
-
100
- Creates the loop, its bounded reprojection executor, and its daemon runner
101
- thread on first call from a given thread. Subsequent calls on the same
102
- thread return the cached loop immediately.
103
-
104
- The loop is stored on ``threading.local`` so each thread (each dask worker,
105
- each Jupyter kernel callback thread) has its own independent loop and
106
- executor — tasks on different threads do not share a pool.
107
-
108
- Using a persistent loop (rather than a fresh one per call) ensures that
109
- any in-flight callbacks from ``async_geotiff``/``obstore`` background
110
- threads can always be delivered, avoiding ``RuntimeError: Event loop is
111
- closed`` errors from callbacks that fire after a fresh loop is torn down.
112
- """
113
- loop: asyncio.AbstractEventLoop | None = getattr(_tls, "loop", None)
114
- if loop is not None and loop.is_running():
115
- return loop
116
-
117
- loop = asyncio.new_event_loop()
118
- loop.set_default_executor(
119
- concurrent.futures.ThreadPoolExecutor(
120
- max_workers=get_max_workers(),
121
- thread_name_prefix="lazycogs-reproject",
122
- ),
123
- )
124
- t = threading.Thread(target=loop.run_forever, daemon=True, name="lazycogs-loop")
125
- t.start()
126
- _tls.loop = loop
127
- return loop
128
-
129
-
130
- def _run_coroutine(coro: CoroutineType) -> Future:
131
- """Run an async coroutine from sync code.
132
-
133
- Submits the coroutine to a persistent per-thread background event loop,
134
- blocking until it completes. The background loop is created on the first
135
- call from a given thread (dask worker, Jupyter kernel thread, etc.) and
136
- reused for all subsequent calls on that same thread.
137
-
138
- Args:
139
- coro: The coroutine to execute.
140
-
141
- Returns:
142
- The return value of the coroutine.
143
-
144
- """
145
- loop = _get_or_create_background_loop()
146
- return asyncio.run_coroutine_threadsafe(coro, loop).result()
147
-
148
-
149
92
  @dataclass
150
93
  class _SpatialWindow:
151
94
  """Resolved spatial indexing window.
@@ -216,7 +159,7 @@ def _resolve_band_indices(
216
159
  return list(range(start, stop, step)), False
217
160
 
218
161
 
219
- def _search_items(
162
+ def _search_items_sync(
220
163
  plan: _ChunkReadPlan,
221
164
  date: str,
222
165
  ) -> list[Any]:
@@ -261,6 +204,28 @@ def _search_items(
261
204
  return items
262
205
 
263
206
 
207
+ async def _search_items_async(
208
+ plan: _ChunkReadPlan,
209
+ date: str,
210
+ ) -> list[Any]:
211
+ """Run the DuckDB search on the dedicated DuckDB executor.
212
+
213
+ DuckDB queries serialise on a single connection internally, so this
214
+ yields the event loop during the query but does not produce parallel
215
+ queries against the same DuckdbClient.
216
+
217
+ Args:
218
+ plan: Read plan carrying all parameters for this chunk.
219
+ date: Acquisition date string (``"YYYY-MM-DD"``).
220
+
221
+ Returns:
222
+ List of STAC items returned by DuckDB.
223
+
224
+ """
225
+ loop = asyncio.get_running_loop()
226
+ return await loop.run_in_executor(_DUCKDB_EXECUTOR, _search_items_sync, plan, date)
227
+
228
+
264
229
  async def _run_one_date(
265
230
  t_idx: int,
266
231
  plan: _ChunkReadPlan,
@@ -268,7 +233,7 @@ async def _run_one_date(
268
233
  """Read and mosaic all COGs for a single time step.
269
234
 
270
235
  Issues one DuckDB query for items overlapping the chunk at this date, then
271
- calls async_mosaic_chunk to fetch and reproject all tiles.
236
+ calls read_chunk_async to fetch and reproject all tiles.
272
237
  Returns None if no items match the query.
273
238
 
274
239
  Args:
@@ -280,14 +245,12 @@ async def _run_one_date(
280
245
 
281
246
  """
282
247
  date = plan.dates[t_idx]
283
-
284
- with plan.duckdb_lock:
285
- items = _search_items(plan, date)
248
+ items = await _search_items_async(plan, date)
286
249
 
287
250
  if not items:
288
251
  return None
289
252
  t0 = time.perf_counter()
290
- chunk_result = await async_mosaic_chunk(
253
+ chunk_result = await read_chunk_async(
291
254
  items=items,
292
255
  bands=plan.selected_bands,
293
256
  chunk_affine=plan.chunk_affine,
@@ -302,7 +265,7 @@ async def _run_one_date(
302
265
  path_fn=plan.path_fn,
303
266
  )
304
267
  logger.debug(
305
- "async_mosaic_chunk bands=%r date=%s (%d items, %dx%d px) took %.3fs",
268
+ "read_chunk_async bands=%r date=%s (%d items, %dx%d px) took %.3fs",
306
269
  plan.selected_bands,
307
270
  date,
308
271
  len(items),
@@ -313,15 +276,17 @@ async def _run_one_date(
313
276
  return chunk_result
314
277
 
315
278
 
316
- async def _run_mosaic_all_dates(
279
+ async def _read_chunk_all_dates(
317
280
  time_indices: list[int],
318
281
  plan: _ChunkReadPlan,
319
282
  ) -> list[dict[str, np.ndarray] | None]:
320
283
  """Run all time steps concurrently inside a single event loop.
321
284
 
322
- DuckDB queries are issued sequentially (the threading.Lock already
323
- serialises access). Mosaic coroutines for all time steps are gathered
324
- concurrently so COG reads and reprojections overlap across time steps.
285
+ DuckDB queries run on the dedicated DuckDB executor; DuckDB itself
286
+ serialises access on a single connection, so concurrent queries on the
287
+ same ``DuckdbClient`` are safe but not parallel. Mosaic coroutines for
288
+ all time steps are gathered concurrently so COG reads and reprojections
289
+ overlap across time steps.
325
290
 
326
291
  Args:
327
292
  time_indices: Ordered list of time-dimension indices to materialise.
@@ -409,12 +374,6 @@ class MultiBandStacBackendArray(BackendArray):
409
374
  path_from_href: Callable[[str], str] | None = field(default=None)
410
375
  shape: tuple[int, ...] = field(init=False)
411
376
  _dst_to_4326: Transformer | None = field(init=False, repr=False, compare=False)
412
- _duckdb_lock: threading.Lock = field(
413
- init=False,
414
- repr=False,
415
- compare=False,
416
- default_factory=threading.Lock,
417
- )
418
377
 
419
378
  def __post_init__(self) -> None:
420
379
  """Derive shape and cache the dst→EPSG:4326 transformer."""
@@ -429,11 +388,6 @@ class MultiBandStacBackendArray(BackendArray):
429
388
  always_xy=True,
430
389
  )
431
390
 
432
- @property
433
- def duckdb_lock(self) -> threading.Lock:
434
- """Lock serialising access to ``duckdb_client``."""
435
- return self._duckdb_lock
436
-
437
391
  def __repr__(self) -> str:
438
392
  """Return a compact string representation."""
439
393
  return f"MultiBandStacBackendArray(bands={self.bands!r}, shape={self.shape})"
@@ -529,16 +483,30 @@ class MultiBandStacBackendArray(BackendArray):
529
483
  key,
530
484
  self.shape,
531
485
  indexing.IndexingSupport.BASIC,
532
- self._raw_getitem,
486
+ self._sync_getitem,
533
487
  )
534
488
 
535
- def _raw_getitem(self, key: tuple[Any, ...]) -> np.ndarray:
489
+ def _sync_getitem(self, key: tuple[Any, ...]) -> np.ndarray:
490
+ """Sync adapter that runs ``_async_getitem`` on the background loop.
491
+
492
+ Args:
493
+ key: A tuple of ``int | slice`` objects for the
494
+ ``(band, time, y, x)`` dimensions.
495
+
496
+ Returns:
497
+ Numpy array with shape determined by the indexing key.
498
+
499
+ """
500
+ return _run_coroutine(self._async_getitem(key))
501
+
502
+ async def _async_getitem(self, key: tuple[Any, ...]) -> np.ndarray:
536
503
  """Materialise the chunk identified by ``key``.
537
504
 
538
- Reads all selected bands together per time step via
539
- :func:`~lazycogs._chunk_reader.async_mosaic_chunk`, issuing
540
- a single DuckDB query per time step and sharing reprojection warp maps
541
- across bands that have identical source geometry.
505
+ Single source of truth for chunk reads. Reads all selected bands
506
+ together per time step via
507
+ :func:`~lazycogs._chunk_reader.read_chunk_async`, issuing a single
508
+ DuckDB query per time step and sharing reprojection warp maps across
509
+ bands that have identical source geometry.
542
510
 
543
511
  Args:
544
512
  key: A tuple of ``int | slice`` objects for the
@@ -562,7 +530,6 @@ class MultiBandStacBackendArray(BackendArray):
562
530
  plan = _ChunkReadPlan(
563
531
  duckdb_client=self.duckdb_client,
564
532
  parquet_path=self.parquet_path,
565
- duckdb_lock=self._duckdb_lock,
566
533
  sortby=self.sortby,
567
534
  filter_expr=self.filter,
568
535
  ids=self.ids,
@@ -582,7 +549,7 @@ class MultiBandStacBackendArray(BackendArray):
582
549
  path_fn=self.path_from_href,
583
550
  )
584
551
 
585
- all_chunk_data = _run_coroutine(_run_mosaic_all_dates(time_indices, plan))
552
+ all_chunk_data = await _read_chunk_all_dates(time_indices, plan)
586
553
 
587
554
  out_shape = (
588
555
  len(band_indices),
@@ -12,6 +12,7 @@ import numpy as np
12
12
  from async_geotiff import GeoTIFF, Overview, RasterArray, Window
13
13
  from numpy import ma
14
14
 
15
+ from lazycogs._executor import _run_coroutine
15
16
  from lazycogs._mosaic_methods import FirstMethod, MosaicMethodBase
16
17
  from lazycogs._reproject import (
17
18
  WarpMap,
@@ -510,7 +511,7 @@ async def _drain_in_order(
510
511
  await asyncio.gather(*tasks, return_exceptions=True)
511
512
 
512
513
 
513
- async def async_mosaic_chunk(
514
+ async def read_chunk_async(
514
515
  items: list[dict],
515
516
  bands: list[str],
516
517
  chunk_affine: Affine,
@@ -527,8 +528,7 @@ async def async_mosaic_chunk(
527
528
  """Read, reproject, and mosaic multiple bands from a list of STAC items.
528
529
 
529
530
  Processes all requested bands together per item so that bands sharing the
530
- same source geometry
531
- compute the reprojection warp map only once (via
531
+ same source geometry compute the reprojection warp map only once (via
532
532
  :func:`_apply_bands_with_warp_cache`).
533
533
 
534
534
  Items are processed in batches of ``max_concurrent_reads``. When all
@@ -615,3 +615,62 @@ async def async_mosaic_chunk(
615
615
  dtype=np.float32,
616
616
  )
617
617
  return output
618
+
619
+
620
+ def read_chunk(
621
+ items: list[dict],
622
+ bands: list[str],
623
+ chunk_affine: Affine,
624
+ dst_crs: CRS,
625
+ chunk_width: int,
626
+ chunk_height: int,
627
+ nodata: float | None = None,
628
+ mosaic_method_cls: type[MosaicMethodBase] | None = None,
629
+ store: ObjectStore | None = None,
630
+ max_concurrent_reads: int = 32,
631
+ warp_cache: dict | None = None,
632
+ path_fn: Callable[[str], str] | None = None,
633
+ ) -> dict[str, np.ndarray]:
634
+ """Run :func:`read_chunk_async` on the persistent per-thread background loop.
635
+
636
+ All arguments are identical to :func:`read_chunk_async`.
637
+
638
+ Args:
639
+ items: List of STAC item dicts to mosaic. Processed in order.
640
+ bands: Asset keys identifying the bands to read from each item.
641
+ chunk_affine: Affine transform of the destination chunk.
642
+ dst_crs: CRS of the destination chunk.
643
+ chunk_width: Width of the destination chunk in pixels.
644
+ chunk_height: Height of the destination chunk in pixels.
645
+ nodata: No-data fill value.
646
+ mosaic_method_cls: Mosaic method class instantiated once per band.
647
+ Defaults to :class:`~lazycogs._mosaic_methods.FirstMethod`.
648
+ store: Optional pre-configured obstore ``ObjectStore`` instance.
649
+ max_concurrent_reads: Maximum number of COG reads to run concurrently.
650
+ warp_cache: Optional cache shared across calls for reusing warp maps
651
+ from earlier time steps.
652
+ path_fn: Optional callable that takes an asset HREF and returns the
653
+ object path to use with *store*.
654
+
655
+ Returns:
656
+ ``dict`` mapping each band name to an array of shape
657
+ ``(cog_bands, chunk_height, chunk_width)`` with dtype matching the
658
+ source COGs.
659
+
660
+ """
661
+ return _run_coroutine(
662
+ read_chunk_async(
663
+ items=items,
664
+ bands=bands,
665
+ chunk_affine=chunk_affine,
666
+ dst_crs=dst_crs,
667
+ chunk_width=chunk_width,
668
+ chunk_height=chunk_height,
669
+ nodata=nodata,
670
+ mosaic_method_cls=mosaic_method_cls,
671
+ store=store,
672
+ max_concurrent_reads=max_concurrent_reads,
673
+ warp_cache=warp_cache,
674
+ path_fn=path_fn,
675
+ ),
676
+ )
@@ -308,7 +308,7 @@ def _build_dataarray(
308
308
  :func:`open` for full documentation.
309
309
 
310
310
  Returns:
311
- Lazy ``xr.DataArray`` with dimensions ``(time, band, y, x)``.
311
+ Lazy ``xr.DataArray`` with dimensions ``(band, time, y, x)``.
312
312
 
313
313
  """
314
314
  dst_affine, dst_width, dst_height, x_coords, y_coords = compute_output_grid(
@@ -386,7 +386,7 @@ def open( # noqa: A001
386
386
  path_from_href: Callable[[str], str] | None = None,
387
387
  duckdb_client: DuckdbClient | None = None,
388
388
  ) -> xr.DataArray:
389
- """Open a mosaic of STAC items as a lazy ``(time, band, y, x)`` DataArray.
389
+ """Open a mosaic of STAC items as a lazy ``(band, time, y, x)`` DataArray.
390
390
 
391
391
  ``href`` must be a path to a geoparquet file (``.parquet`` or
392
392
  ``.geoparquet``) or, when *duckdb_client* is provided, to a
@@ -480,7 +480,7 @@ def open( # noqa: A001
480
480
  )
481
481
 
482
482
  Returns:
483
- Lazy ``xr.DataArray`` with dimensions ``(time, band, y, x)``.
483
+ Lazy ``xr.DataArray`` with dimensions ``(band, time, y, x)``.
484
484
 
485
485
  Raises:
486
486
  ValueError: If ``href`` is not a ``.parquet`` or ``.geoparquet`` file
@@ -0,0 +1,128 @@
1
+ """Thread pool and event loop configuration for reprojection and DuckDB work."""
2
+
3
+ from __future__ import annotations
4
+
5
+ import asyncio
6
+ import concurrent.futures
7
+ import os
8
+ import threading
9
+ from typing import TYPE_CHECKING
10
+
11
+ if TYPE_CHECKING:
12
+ from asyncio.futures import Future
13
+ from types import CoroutineType
14
+
15
+ config: dict[str, int | None] = {"max_workers": None}
16
+
17
+ _tls = threading.local()
18
+
19
+ # DuckDB queries serialise on a single connection, so a small pool is enough.
20
+ # Kept separate from the reprojection executor so a long reprojection cannot
21
+ # starve a queued query within the same chunk read.
22
+ _DUCKDB_EXECUTOR = concurrent.futures.ThreadPoolExecutor(
23
+ max_workers=2,
24
+ thread_name_prefix="lazycogs-duckdb",
25
+ )
26
+
27
+
28
+ def _default_workers() -> int:
29
+ """Return the default worker count: CPUs up to a cap of 4.
30
+
31
+ Reprojection (pyproj + numpy) is memory-bandwidth-bound, not compute-bound.
32
+ Benchmarks show diminishing returns beyond 4 concurrent threads because they
33
+ saturate the memory bus rather than adding CPU throughput. Keep the default
34
+ conservative.
35
+ """
36
+ return min(os.cpu_count() or 4, 4)
37
+
38
+
39
+ def get_max_workers() -> int:
40
+ """Return the configured worker count, or the default if not set.
41
+
42
+ Returns:
43
+ Number of reprojection threads each event loop will use.
44
+
45
+ """
46
+ val = config["max_workers"]
47
+ return val if val is not None else _default_workers()
48
+
49
+
50
+ def set_reproject_workers(n: int) -> None:
51
+ """Set the number of threads each thread's event loop uses for reprojection.
52
+
53
+ Each thread (dask worker, Jupyter kernel callback thread, etc.) gets one
54
+ persistent background event loop with one bounded reprojection
55
+ ``ThreadPoolExecutor``. All chunk reads on that thread share the same loop
56
+ and executor. Dask tasks on different threads do not compete for a shared
57
+ pool. Total reprojection threads at any moment is at most
58
+ ``n x active_thread_count``.
59
+
60
+ Reprojection is memory-bandwidth-bound rather than compute-bound, so values
61
+ above 4 typically offer no benefit and can hurt throughput due to memory
62
+ contention. The default is ``min(os.cpu_count(), 4)``.
63
+
64
+ To improve overall throughput, prefer adding time or band parallelism via
65
+ dask (``chunks={"time": 1}``) over raising this value.
66
+
67
+ Args:
68
+ n: Number of worker threads per event loop. Must be >= 1.
69
+
70
+ Raises:
71
+ ValueError: If ``n`` is less than 1.
72
+
73
+ """
74
+ if n < 1:
75
+ raise ValueError(f"n must be >= 1, got {n!r}")
76
+ config["max_workers"] = n
77
+
78
+
79
+ def _get_or_create_background_loop() -> asyncio.AbstractEventLoop:
80
+ """Return the persistent background event loop for the current thread.
81
+
82
+ Creates the loop, its bounded reprojection executor, and its daemon runner
83
+ thread on first call from a given thread. Subsequent calls on the same
84
+ thread return the cached loop immediately.
85
+
86
+ The loop is stored on ``threading.local`` so each thread (each dask worker,
87
+ each Jupyter kernel callback thread) has its own independent loop and
88
+ executor — tasks on different threads do not share a pool.
89
+
90
+ Using a persistent loop (rather than a fresh one per call) ensures that
91
+ any in-flight callbacks from ``async_geotiff``/``obstore`` background
92
+ threads can always be delivered, avoiding ``RuntimeError: Event loop is
93
+ closed`` errors from callbacks that fire after a fresh loop is torn down.
94
+ """
95
+ loop: asyncio.AbstractEventLoop | None = getattr(_tls, "loop", None)
96
+ if loop is not None and loop.is_running():
97
+ return loop
98
+
99
+ loop = asyncio.new_event_loop()
100
+ loop.set_default_executor(
101
+ concurrent.futures.ThreadPoolExecutor(
102
+ max_workers=get_max_workers(),
103
+ thread_name_prefix="lazycogs-reproject",
104
+ ),
105
+ )
106
+ t = threading.Thread(target=loop.run_forever, daemon=True, name="lazycogs-loop")
107
+ t.start()
108
+ _tls.loop = loop
109
+ return loop
110
+
111
+
112
+ def _run_coroutine(coro: CoroutineType) -> Future:
113
+ """Run an async coroutine from sync code.
114
+
115
+ Submits the coroutine to a persistent per-thread background event loop,
116
+ blocking until it completes. The background loop is created on the first
117
+ call from a given thread (dask worker, Jupyter kernel thread, etc.) and
118
+ reused for all subsequent calls on that same thread.
119
+
120
+ Args:
121
+ coro: The coroutine to execute.
122
+
123
+ Returns:
124
+ The return value of the coroutine.
125
+
126
+ """
127
+ loop = _get_or_create_background_loop()
128
+ return asyncio.run_coroutine_threadsafe(coro, loop).result()
@@ -14,14 +14,16 @@ from affine import Affine
14
14
  from pandas import DataFrame
15
15
  from pyproj import CRS, Transformer
16
16
 
17
- from lazycogs._backend import MultiBandStacBackendArray, _run_coroutine
18
17
  from lazycogs._chunk_reader import _ChunkContext, _open_and_window
18
+ from lazycogs._executor import _DUCKDB_EXECUTOR, _run_coroutine
19
19
 
20
20
  if TYPE_CHECKING:
21
21
  from collections.abc import Iterator
22
22
 
23
23
  from obstore.store import ObjectStore
24
24
 
25
+ from lazycogs._backend import MultiBandStacBackendArray
26
+
25
27
  logger = logging.getLogger(__name__)
26
28
 
27
29
  _COG_MULTI_THRESHOLD = 2
@@ -505,9 +507,9 @@ async def _explain_async(
505
507
  Issues one DuckDB query per ``(time step, spatial tile)`` — not one per
506
508
  ``(band, time step, spatial tile)`` — because the query result is
507
509
  band-independent. All ``(time x tile)`` queries are dispatched
508
- concurrently via :func:`asyncio.gather`; the ``backend.duckdb_lock``
509
- serialises actual DuckDB access. Each query result is then fanned across
510
- all active bands to produce one :class:`ChunkRead` per
510
+ concurrently via :func:`asyncio.gather` so the per-query JSON
511
+ construction and result processing overlap. Each query result is then
512
+ fanned across all active bands to produce one :class:`ChunkRead` per
511
513
  ``(band, time, tile)`` combination.
512
514
 
513
515
  Args:
@@ -598,15 +600,18 @@ async def _explain_async(
598
600
  actual_h,
599
601
  dst_crs,
600
602
  )
601
- with backend.duckdb_lock:
602
- items = backend.duckdb_client.search(
603
+ loop = asyncio.get_running_loop()
604
+ items = await loop.run_in_executor(
605
+ _DUCKDB_EXECUTOR,
606
+ lambda: backend.duckdb_client.search(
603
607
  backend.parquet_path,
604
608
  bbox=chunk_bbox_4326,
605
609
  datetime=date_filter,
606
610
  sortby=backend.sortby,
607
611
  filter=backend.filter,
608
612
  ids=backend.ids,
609
- )
613
+ ),
614
+ )
610
615
  logger.debug(
611
616
  "explain date=%s chunk=(%d,%d) -> %d items (for %d band(s))",
612
617
  date_filter,
@@ -1,57 +0,0 @@
1
- """Configure the per-event-loop thread pool size for CPU-bound reprojection work."""
2
-
3
- from __future__ import annotations
4
-
5
- import os
6
-
7
- config: dict[str, int | None] = {"max_workers": None}
8
-
9
-
10
- def _default_workers() -> int:
11
- """Return the default worker count: CPUs up to a cap of 4.
12
-
13
- Reprojection (pyproj + numpy) is memory-bandwidth-bound, not compute-bound.
14
- Benchmarks show diminishing returns beyond 4 concurrent threads because they
15
- saturate the memory bus rather than adding CPU throughput. Keep the default
16
- conservative.
17
- """
18
- return min(os.cpu_count() or 4, 4)
19
-
20
-
21
- def get_max_workers() -> int:
22
- """Return the configured worker count, or the default if not set.
23
-
24
- Returns:
25
- Number of reprojection threads each event loop will use.
26
-
27
- """
28
- val = config["max_workers"]
29
- return val if val is not None else _default_workers()
30
-
31
-
32
- def set_reproject_workers(n: int) -> None:
33
- """Set the number of threads each chunk's event loop uses for reprojection.
34
-
35
- Each chunk read creates a fresh asyncio event loop with its own dedicated
36
- ``ThreadPoolExecutor`` bounded to ``n`` workers. Dask tasks do not compete
37
- for a shared pool — each task gets ``n`` independent reprojection threads.
38
- Total reprojection threads at any moment is at most
39
- ``n x dask_worker_count``.
40
-
41
- Reprojection is memory-bandwidth-bound rather than compute-bound, so values
42
- above 4 typically offer no benefit and can hurt throughput due to memory
43
- contention. The default is ``min(os.cpu_count(), 4)``.
44
-
45
- To improve overall throughput, prefer adding time or band parallelism via
46
- dask (``chunks={"time": 1}``) over raising this value.
47
-
48
- Args:
49
- n: Number of worker threads per event loop. Must be >= 1.
50
-
51
- Raises:
52
- ValueError: If ``n`` is less than 1.
53
-
54
- """
55
- if n < 1:
56
- raise ValueError(f"n must be >= 1, got {n!r}")
57
- config["max_workers"] = n
File without changes
File without changes
File without changes