easy-eo 0.4.2__tar.gz → 0.5.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {easy_eo-0.4.2 → easy_eo-0.5.0}/PKG-INFO +8 -3
- {easy_eo-0.4.2 → easy_eo-0.5.0}/README.md +3 -2
- {easy_eo-0.4.2 → easy_eo-0.5.0}/eeo/__init__.py +1 -1
- {easy_eo-0.4.2 → easy_eo-0.5.0}/eeo/_optional.py +8 -5
- {easy_eo-0.4.2 → easy_eo-0.5.0}/eeo/common.py +67 -0
- easy_eo-0.5.0/eeo/core/adapters/__init__.py +15 -0
- {easy_eo-0.4.2 → easy_eo-0.5.0}/eeo/core/adapters/base.py +4 -3
- {easy_eo-0.4.2 → easy_eo-0.5.0}/eeo/core/adapters/rasterio.py +3 -1
- easy_eo-0.5.0/eeo/core/adapters/xarray.py +393 -0
- {easy_eo-0.4.2 → easy_eo-0.5.0}/eeo/core/blockwise.py +15 -4
- {easy_eo-0.4.2 → easy_eo-0.5.0}/eeo/core/core.py +60 -17
- {easy_eo-0.4.2 → easy_eo-0.5.0}/eeo/core/core.pyi +2 -2
- {easy_eo-0.4.2 → easy_eo-0.5.0}/eeo/core/loader.py +34 -10
- easy_eo-0.5.0/eeo/core/remote.py +122 -0
- {easy_eo-0.4.2 → easy_eo-0.5.0}/eeo/core/types.py +4 -0
- {easy_eo-0.4.2 → easy_eo-0.5.0}/eeo/io/_stacking.py +1 -11
- {easy_eo-0.4.2 → easy_eo-0.5.0}/eeo/ops/merge.py +9 -16
- {easy_eo-0.4.2 → easy_eo-0.5.0}/eeo/preprocessing/clip.py +4 -14
- {easy_eo-0.4.2 → easy_eo-0.5.0}/eeo/preprocessing/reproject.py +3 -8
- {easy_eo-0.4.2 → easy_eo-0.5.0}/eeo/viz/plot.py +8 -1
- {easy_eo-0.4.2 → easy_eo-0.5.0}/pyproject.toml +9 -1
- easy_eo-0.4.2/eeo/core/adapters/__init__.py +0 -11
- {easy_eo-0.4.2 → easy_eo-0.5.0}/.gitignore +0 -0
- {easy_eo-0.4.2 → easy_eo-0.5.0}/LICENSE +0 -0
- {easy_eo-0.4.2 → easy_eo-0.5.0}/eeo/_show_versions.py +0 -0
- {easy_eo-0.4.2 → easy_eo-0.5.0}/eeo/analysis/__init__.py +0 -0
- {easy_eo-0.4.2 → easy_eo-0.5.0}/eeo/analysis/indices.py +0 -0
- {easy_eo-0.4.2 → easy_eo-0.5.0}/eeo/analysis/stats.py +0 -0
- {easy_eo-0.4.2 → easy_eo-0.5.0}/eeo/core/__init__.py +0 -0
- {easy_eo-0.4.2 → easy_eo-0.5.0}/eeo/core/adapters/numpy.py +0 -0
- {easy_eo-0.4.2 → easy_eo-0.5.0}/eeo/core/decorators.py +0 -0
- {easy_eo-0.4.2 → easy_eo-0.5.0}/eeo/core/exceptions.py +0 -0
- {easy_eo-0.4.2 → easy_eo-0.5.0}/eeo/core/plugins.py +0 -0
- {easy_eo-0.4.2 → easy_eo-0.5.0}/eeo/core/streaming.py +0 -0
- {easy_eo-0.4.2 → easy_eo-0.5.0}/eeo/datasets/__init__.py +0 -0
- {easy_eo-0.4.2 → easy_eo-0.5.0}/eeo/datasets/_cache.py +0 -0
- {easy_eo-0.4.2 → easy_eo-0.5.0}/eeo/datasets/_registry.py +0 -0
- {easy_eo-0.4.2 → easy_eo-0.5.0}/eeo/datasets/_samples.py +0 -0
- {easy_eo-0.4.2 → easy_eo-0.5.0}/eeo/io/__init__.py +0 -0
- {easy_eo-0.4.2 → easy_eo-0.5.0}/eeo/io/_archive.py +0 -0
- {easy_eo-0.4.2 → easy_eo-0.5.0}/eeo/io/_bands.py +0 -0
- {easy_eo-0.4.2 → easy_eo-0.5.0}/eeo/io/_landsat.py +0 -0
- {easy_eo-0.4.2 → easy_eo-0.5.0}/eeo/io/_sentinel2.py +0 -0
- {easy_eo-0.4.2 → easy_eo-0.5.0}/eeo/io/products.py +0 -0
- {easy_eo-0.4.2 → easy_eo-0.5.0}/eeo/io/stac.py +0 -0
- {easy_eo-0.4.2 → easy_eo-0.5.0}/eeo/io/xarray.py +0 -0
- {easy_eo-0.4.2 → easy_eo-0.5.0}/eeo/ops/__init__.py +0 -0
- {easy_eo-0.4.2 → easy_eo-0.5.0}/eeo/ops/algebra.py +0 -0
- {easy_eo-0.4.2 → easy_eo-0.5.0}/eeo/preprocessing/__init__.py +0 -0
- {easy_eo-0.4.2 → easy_eo-0.5.0}/eeo/preprocessing/masking.py +0 -0
- {easy_eo-0.4.2 → easy_eo-0.5.0}/eeo/preprocessing/normalize.py +0 -0
- {easy_eo-0.4.2 → easy_eo-0.5.0}/eeo/preprocessing/quality.py +0 -0
- {easy_eo-0.4.2 → easy_eo-0.5.0}/eeo/preprocessing/resample.py +0 -0
- {easy_eo-0.4.2 → easy_eo-0.5.0}/eeo/py.typed +0 -0
- {easy_eo-0.4.2 → easy_eo-0.5.0}/eeo/viz/__init__.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.5
|
|
2
2
|
Name: easy-eo
|
|
3
|
-
Version: 0.
|
|
3
|
+
Version: 0.5.0
|
|
4
4
|
Summary: A lightweight Earth Observation utilities library for chainable raster operations
|
|
5
5
|
Project-URL: Homepage, https://github.com/tommy-burns/easy-eo
|
|
6
6
|
Project-URL: Repository, https://github.com/tommy-burns/easy-eo
|
|
@@ -52,6 +52,10 @@ Requires-Dist: pre-commit>=3.7; extra == 'dev'
|
|
|
52
52
|
Requires-Dist: pytest-cov>=5.0; extra == 'dev'
|
|
53
53
|
Requires-Dist: pytest>=8.0; extra == 'dev'
|
|
54
54
|
Requires-Dist: ruff>=0.6; extra == 'dev'
|
|
55
|
+
Provides-Extra: lazy
|
|
56
|
+
Requires-Dist: dask[array]>=2024.8; extra == 'lazy'
|
|
57
|
+
Requires-Dist: rioxarray<1,>=0.17; extra == 'lazy'
|
|
58
|
+
Requires-Dist: xarray>=2024.7; extra == 'lazy'
|
|
55
59
|
Provides-Extra: stac
|
|
56
60
|
Requires-Dist: planetary-computer<2,>=1.0; extra == 'stac'
|
|
57
61
|
Requires-Dist: pystac-client<1,>=0.8; extra == 'stac'
|
|
@@ -253,13 +257,14 @@ conda install -c conda-forge easy-eo
|
|
|
253
257
|
```
|
|
254
258
|
|
|
255
259
|
That is everything you need for the core: raster I/O, algebra, indices,
|
|
256
|
-
preprocessing and plotting.
|
|
257
|
-
only install them if you use them:
|
|
260
|
+
preprocessing and plotting. Three heavier integrations are kept separate, so
|
|
261
|
+
you only install them if you use them:
|
|
258
262
|
|
|
259
263
|
| Adds | pip | conda |
|
|
260
264
|
| --- | --- | --- |
|
|
261
265
|
| `stac_search()` and loading scenes from STAC catalogs | `pip install "easy-eo[stac]"` | `conda install -c conda-forge easy-eo pystac-client planetary-computer` |
|
|
262
266
|
| `to_xarray()` / `from_xarray()` | `pip install "easy-eo[xarray]"` | `conda install -c conda-forge easy-eo xarray rioxarray` |
|
|
267
|
+
| Opening a raster lazily (`load_raster(path, chunks=...)`), including over HTTP | `pip install "easy-eo[lazy]"` | `conda install -c conda-forge easy-eo xarray rioxarray dask-core` |
|
|
263
268
|
|
|
264
269
|
pip extras compose - `pip install "easy-eo[stac,xarray]"` installs both. conda
|
|
265
270
|
has no concept of extras, so `conda install "easy-eo[stac]"` is not a valid
|
|
@@ -191,13 +191,14 @@ conda install -c conda-forge easy-eo
|
|
|
191
191
|
```
|
|
192
192
|
|
|
193
193
|
That is everything you need for the core: raster I/O, algebra, indices,
|
|
194
|
-
preprocessing and plotting.
|
|
195
|
-
only install them if you use them:
|
|
194
|
+
preprocessing and plotting. Three heavier integrations are kept separate, so
|
|
195
|
+
you only install them if you use them:
|
|
196
196
|
|
|
197
197
|
| Adds | pip | conda |
|
|
198
198
|
| --- | --- | --- |
|
|
199
199
|
| `stac_search()` and loading scenes from STAC catalogs | `pip install "easy-eo[stac]"` | `conda install -c conda-forge easy-eo pystac-client planetary-computer` |
|
|
200
200
|
| `to_xarray()` / `from_xarray()` | `pip install "easy-eo[xarray]"` | `conda install -c conda-forge easy-eo xarray rioxarray` |
|
|
201
|
+
| Opening a raster lazily (`load_raster(path, chunks=...)`), including over HTTP | `pip install "easy-eo[lazy]"` | `conda install -c conda-forge easy-eo xarray rioxarray dask-core` |
|
|
201
202
|
|
|
202
203
|
pip extras compose - `pip install "easy-eo[stac,xarray]"` installs both. conda
|
|
203
204
|
has no concept of extras, so `conda install "easy-eo[stac]"` is not a valid
|
|
@@ -24,14 +24,17 @@ from .core.exceptions import MissingDependencyError
|
|
|
24
24
|
# list with nothing to opt into, and `conda install "easy-eo[stac]"` does not
|
|
25
25
|
# merely miss the extra — it fails to parse, because brackets already mean
|
|
26
26
|
# key-value constraints in conda's match syntax. So a conda user installs the
|
|
27
|
-
# same packages by name. The names
|
|
28
|
-
#
|
|
29
|
-
# `
|
|
30
|
-
#
|
|
31
|
-
#
|
|
27
|
+
# same packages by name. The names usually equal the PyPI ones, but that is not
|
|
28
|
+
# a rule: conda-forge's `dask` is a metapackage that also pulls in `distributed`
|
|
29
|
+
# and `bokeh`, so the lazy extra maps to `dask-core`, the equivalent of pip's
|
|
30
|
+
# `dask[array]` (as Matplotlib is `matplotlib-base`). That is why the mapping is
|
|
31
|
+
# written out rather than derived. tests/test_optional_dependencies.py checks it
|
|
32
|
+
# covers every declared extra, so a new extra cannot ship without its conda
|
|
33
|
+
# equivalent.
|
|
32
34
|
_CONDA_PACKAGES: dict[str, tuple[str, ...]] = {
|
|
33
35
|
"stac": ("pystac-client", "planetary-computer"),
|
|
34
36
|
"xarray": ("xarray", "rioxarray"),
|
|
37
|
+
"lazy": ("xarray", "rioxarray", "dask-core"),
|
|
35
38
|
}
|
|
36
39
|
|
|
37
40
|
|
|
@@ -40,6 +40,73 @@ def is_rasterio_backed(ds: EEORasterDataset) -> bool:
|
|
|
40
40
|
return isinstance(ds._adapter, RasterioAdapter)
|
|
41
41
|
|
|
42
42
|
|
|
43
|
+
def require_rasterio(ds: EEORasterDataset, operation: str) -> EEORasterDataset:
|
|
44
|
+
"""Return ``ds`` on the rasterio backend, for an op that needs one.
|
|
45
|
+
|
|
46
|
+
A lazy dataset is promoted, which for a file-backed one reopens the file
|
|
47
|
+
and reads nothing. A NumPy-backed one is refused rather than promoted
|
|
48
|
+
silently: its pixels are already in memory and promoting copies all of
|
|
49
|
+
them, so that is the caller's decision to make.
|
|
50
|
+
|
|
51
|
+
Parameters
|
|
52
|
+
----------
|
|
53
|
+
ds : EEORasterDataset
|
|
54
|
+
Dataset the operation was called on.
|
|
55
|
+
operation : str
|
|
56
|
+
Name of the operation, used in the error message.
|
|
57
|
+
|
|
58
|
+
Returns
|
|
59
|
+
-------
|
|
60
|
+
EEORasterDataset
|
|
61
|
+
``ds`` itself when it is rasterio-backed, else a promoted copy.
|
|
62
|
+
|
|
63
|
+
Raises
|
|
64
|
+
------
|
|
65
|
+
BackendError
|
|
66
|
+
If ``ds`` uses the NumPy backend.
|
|
67
|
+
"""
|
|
68
|
+
from eeo.core.exceptions import BackendError
|
|
69
|
+
|
|
70
|
+
if is_rasterio_backed(ds):
|
|
71
|
+
return ds
|
|
72
|
+
if _is_promotable_without_reading(ds):
|
|
73
|
+
return ds.to_rasterio()
|
|
74
|
+
raise BackendError(
|
|
75
|
+
f"{operation} requires a rasterio-backed dataset; this dataset uses the "
|
|
76
|
+
"NumPy backend. Call .to_rasterio() first."
|
|
77
|
+
)
|
|
78
|
+
|
|
79
|
+
|
|
80
|
+
def promote_for_decimated_read(ds: EEORasterDataset) -> EEORasterDataset:
|
|
81
|
+
"""Return a dataset that can serve a decimated (``out_shape``) read.
|
|
82
|
+
|
|
83
|
+
Only the rasterio backend reads decimated. A lazy dataset reaches it by
|
|
84
|
+
reopening its file, which is what keeps plotting a large lazy raster from
|
|
85
|
+
reading it whole. Anything else is returned unchanged, for the caller to
|
|
86
|
+
fall back on a full read.
|
|
87
|
+
|
|
88
|
+
Parameters
|
|
89
|
+
----------
|
|
90
|
+
ds : EEORasterDataset
|
|
91
|
+
Dataset about to be read for display.
|
|
92
|
+
|
|
93
|
+
Returns
|
|
94
|
+
-------
|
|
95
|
+
EEORasterDataset
|
|
96
|
+
``ds``, or a rasterio-backed promotion of it that cost no read.
|
|
97
|
+
"""
|
|
98
|
+
if not is_rasterio_backed(ds) and _is_promotable_without_reading(ds):
|
|
99
|
+
return ds.to_rasterio()
|
|
100
|
+
return ds
|
|
101
|
+
|
|
102
|
+
|
|
103
|
+
def _is_promotable_without_reading(ds: EEORasterDataset) -> bool:
|
|
104
|
+
"""Report whether ``ds`` can reach the rasterio backend without a read."""
|
|
105
|
+
from eeo.core.adapters import XarrayAdapter
|
|
106
|
+
|
|
107
|
+
return isinstance(ds._adapter, XarrayAdapter) and ds._adapter.source_path is not None
|
|
108
|
+
|
|
109
|
+
|
|
43
110
|
def normalize_resampling_method(value):
|
|
44
111
|
"""Normalize a resampling method to a ``rasterio.enums.Resampling`` value."""
|
|
45
112
|
from eeo.core.exceptions import ValidationError
|
|
@@ -0,0 +1,15 @@
|
|
|
1
|
+
"""Backend adapters abstracting NumPy-, rasterio- and xarray-backed rasters."""
|
|
2
|
+
|
|
3
|
+
from .base import BaseRasterAdapter
|
|
4
|
+
from .numpy import NumpyRasterioAdapter
|
|
5
|
+
from .rasterio import RasterioAdapter
|
|
6
|
+
|
|
7
|
+
# Importing the module does not import xarray; the adapter does so on use.
|
|
8
|
+
from .xarray import XarrayAdapter
|
|
9
|
+
|
|
10
|
+
__all__ = [
|
|
11
|
+
"BaseRasterAdapter",
|
|
12
|
+
"RasterioAdapter",
|
|
13
|
+
"NumpyRasterioAdapter",
|
|
14
|
+
"XarrayAdapter",
|
|
15
|
+
]
|
|
@@ -16,8 +16,8 @@ from eeo.core.types import StrPath
|
|
|
16
16
|
class BaseRasterAdapter(ABC):
|
|
17
17
|
"""Backend-agnostic interface that every raster backend must implement.
|
|
18
18
|
|
|
19
|
-
Concrete adapters (NumPy-backed, rasterio-backed, and
|
|
20
|
-
|
|
19
|
+
Concrete adapters (NumPy-backed, rasterio-backed, and the lazy
|
|
20
|
+
xarray-backed one) implement these methods so that operations in ``eeo`` stay
|
|
21
21
|
backend-agnostic. Metadata accessors return rasterio/affine types
|
|
22
22
|
regardless of the underlying backend.
|
|
23
23
|
"""
|
|
@@ -122,7 +122,8 @@ class BaseRasterAdapter(ABC):
|
|
|
122
122
|
def backend(self) -> Any:
|
|
123
123
|
"""Return the underlying backend object.
|
|
124
124
|
|
|
125
|
-
Returns the raw ``rasterio.DatasetReader
|
|
125
|
+
Returns the raw ``rasterio.DatasetReader``, ``numpy.ndarray`` or
|
|
126
|
+
``xarray.DataArray``.
|
|
126
127
|
This bypasses Easy-EO's abstractions; use it only for interop.
|
|
127
128
|
"""
|
|
128
129
|
...
|
|
@@ -9,6 +9,7 @@ import rasterio as rio
|
|
|
9
9
|
from rasterio.io import DatasetReader, DatasetWriter, MemoryFile
|
|
10
10
|
|
|
11
11
|
from eeo.core.exceptions import BackendError
|
|
12
|
+
from eeo.core.remote import open_env
|
|
12
13
|
from eeo.core.types import StrPath
|
|
13
14
|
|
|
14
15
|
from .base import BaseRasterAdapter
|
|
@@ -47,7 +48,8 @@ class RasterioAdapter(BaseRasterAdapter):
|
|
|
47
48
|
@classmethod
|
|
48
49
|
def from_path(cls, path: StrPath) -> RasterioAdapter:
|
|
49
50
|
try:
|
|
50
|
-
|
|
51
|
+
with open_env(path):
|
|
52
|
+
dataset = rio.open(path)
|
|
51
53
|
except Exception as e:
|
|
52
54
|
raise BackendError(f"failed to open raster: {path}") from e
|
|
53
55
|
return cls(dataset)
|
|
@@ -0,0 +1,393 @@
|
|
|
1
|
+
"""Lazy, dask-chunked raster adapter backed by xarray."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import itertools
|
|
6
|
+
from collections.abc import Sequence
|
|
7
|
+
from typing import Any
|
|
8
|
+
|
|
9
|
+
import numpy as np
|
|
10
|
+
import rasterio as rio
|
|
11
|
+
from rasterio.coords import BoundingBox
|
|
12
|
+
from rasterio.crs import CRS
|
|
13
|
+
from rasterio.transform import Affine, array_bounds
|
|
14
|
+
from rasterio.windows import Window
|
|
15
|
+
|
|
16
|
+
from eeo._optional import import_optional
|
|
17
|
+
from eeo.core.exceptions import BackendError, ValidationError
|
|
18
|
+
from eeo.core.remote import open_env
|
|
19
|
+
from eeo.core.types import ChunkSpec, StrPath
|
|
20
|
+
|
|
21
|
+
from .base import BaseRasterAdapter
|
|
22
|
+
|
|
23
|
+
_PURPOSE = "the lazy backend"
|
|
24
|
+
_DIMS = ("band", "y", "x")
|
|
25
|
+
_CHUNK_DIMS = frozenset(_DIMS)
|
|
26
|
+
|
|
27
|
+
|
|
28
|
+
def _import_rioxarray() -> Any:
|
|
29
|
+
"""Import rioxarray, which also registers the ``.rio`` accessor."""
|
|
30
|
+
return import_optional("rioxarray", extra="lazy", purpose=_PURPOSE)
|
|
31
|
+
|
|
32
|
+
|
|
33
|
+
def _auto_chunk_sizes(src: Any) -> dict[str, int]:
|
|
34
|
+
"""Return the chunk sizes ``chunks="auto"`` means for an open raster.
|
|
35
|
+
|
|
36
|
+
The same sizes :func:`rioxarray.open_rasterio` computes for ``"auto"``:
|
|
37
|
+
what dask would pick for the configured chunk size, snapped to the file's
|
|
38
|
+
own internal blocks. They are computed here so they can be passed as a
|
|
39
|
+
dict keyed by dimension. rioxarray passes its own as a dimension-order
|
|
40
|
+
tuple, which xarray deprecated and warns about — a warning its callers can
|
|
41
|
+
do nothing about, and which only versions needing Python 3.12 have stopped
|
|
42
|
+
emitting.
|
|
43
|
+
|
|
44
|
+
Parameters
|
|
45
|
+
----------
|
|
46
|
+
src : rasterio.io.DatasetReader
|
|
47
|
+
The open raster, for its shape, dtype and block layout.
|
|
48
|
+
|
|
49
|
+
Returns
|
|
50
|
+
-------
|
|
51
|
+
dict of str to int
|
|
52
|
+
Chunk size for each of ``"band"``, ``"y"`` and ``"x"``.
|
|
53
|
+
"""
|
|
54
|
+
dask_array = import_optional("dask.array", extra="lazy", purpose=_PURPOSE)
|
|
55
|
+
# One band per chunk, as rioxarray does: bands are stored separately, so a
|
|
56
|
+
# chunk spanning several of them would read from several places at once.
|
|
57
|
+
blocks = dask_array.core.normalize_chunks(
|
|
58
|
+
chunks=(1, "auto", "auto"),
|
|
59
|
+
shape=(src.count, src.height, src.width),
|
|
60
|
+
dtype=np.dtype(src.dtypes[0]),
|
|
61
|
+
previous_chunks=tuple((size,) for size in (1, *src.block_shapes[0])),
|
|
62
|
+
)
|
|
63
|
+
# normalize_chunks returns every block's size per dimension, regular apart
|
|
64
|
+
# from a possibly short last one; the first is the size that generates them.
|
|
65
|
+
return {dim: int(sizes[0]) for dim, sizes in zip(_DIMS, blocks, strict=True) if sizes}
|
|
66
|
+
|
|
67
|
+
|
|
68
|
+
def validate_chunks(chunks: object) -> ChunkSpec:
|
|
69
|
+
"""Check a chunk specification before it reaches dask.
|
|
70
|
+
|
|
71
|
+
Parameters
|
|
72
|
+
----------
|
|
73
|
+
chunks : object
|
|
74
|
+
Candidate chunk specification.
|
|
75
|
+
|
|
76
|
+
Returns
|
|
77
|
+
-------
|
|
78
|
+
str or int or dict
|
|
79
|
+
``chunks`` unchanged, once validated.
|
|
80
|
+
|
|
81
|
+
Raises
|
|
82
|
+
------
|
|
83
|
+
ValidationError
|
|
84
|
+
If ``chunks`` is not ``"auto"``, a positive int, or a dict mapping
|
|
85
|
+
some of ``"band"``, ``"y"``, ``"x"`` to a positive int or ``"auto"``.
|
|
86
|
+
"""
|
|
87
|
+
|
|
88
|
+
def valid_size(value: object) -> bool:
|
|
89
|
+
if value == "auto":
|
|
90
|
+
return True
|
|
91
|
+
return isinstance(value, int) and not isinstance(value, bool) and value > 0
|
|
92
|
+
|
|
93
|
+
if isinstance(chunks, dict):
|
|
94
|
+
unknown = set(chunks) - _CHUNK_DIMS
|
|
95
|
+
if not unknown and all(valid_size(v) for v in chunks.values()):
|
|
96
|
+
return chunks
|
|
97
|
+
elif valid_size(chunks):
|
|
98
|
+
return chunks # type: ignore[return-value]
|
|
99
|
+
raise ValidationError(
|
|
100
|
+
"chunks must be 'auto', a positive int, or a dict mapping 'band', 'y' "
|
|
101
|
+
f"and/or 'x' to a positive int or 'auto'; got {chunks!r}"
|
|
102
|
+
)
|
|
103
|
+
|
|
104
|
+
|
|
105
|
+
class XarrayAdapter(BaseRasterAdapter):
|
|
106
|
+
"""Lazy raster adapter over a dask-chunked :class:`xarray.DataArray`.
|
|
107
|
+
|
|
108
|
+
Opening a raster through this adapter reads metadata only; pixels stay in
|
|
109
|
+
the file until something asks for them, and then only the requested bands
|
|
110
|
+
and window are computed. Needs the optional ``lazy`` extra
|
|
111
|
+
(``pip install "easy-eo[lazy]"``).
|
|
112
|
+
|
|
113
|
+
Prefer :meth:`from_path` over calling the constructor directly.
|
|
114
|
+
|
|
115
|
+
Parameters
|
|
116
|
+
----------
|
|
117
|
+
dataarray : xarray.DataArray
|
|
118
|
+
Georeferenced array with dimensions ``("band", "y", "x")``, laid out
|
|
119
|
+
as :func:`rioxarray.open_rasterio` returns it, holding raw stored values
|
|
120
|
+
(not masked or scaled).
|
|
121
|
+
driver : str, default "GTiff"
|
|
122
|
+
GDAL driver of the source, reported in the metadata.
|
|
123
|
+
source_path : str or path-like or None, default None
|
|
124
|
+
File the array was opened from, when it was opened from one. It is
|
|
125
|
+
what lets a lazy dataset be promoted to the rasterio backend by
|
|
126
|
+
reopening the file rather than by reading every pixel.
|
|
127
|
+
|
|
128
|
+
Raises
|
|
129
|
+
------
|
|
130
|
+
MissingDependencyError
|
|
131
|
+
If the ``lazy`` extra is not installed.
|
|
132
|
+
ValidationError
|
|
133
|
+
If ``dataarray`` does not have dimensions ``("band", "y", "x")``.
|
|
134
|
+
"""
|
|
135
|
+
|
|
136
|
+
def __init__(
|
|
137
|
+
self, dataarray: Any, *, driver: str = "GTiff", source_path: StrPath | None = None
|
|
138
|
+
) -> None:
|
|
139
|
+
_import_rioxarray()
|
|
140
|
+
if tuple(dataarray.dims) != _DIMS:
|
|
141
|
+
raise ValidationError(
|
|
142
|
+
f"dataarray must have dimensions {_DIMS}; got {tuple(dataarray.dims)}"
|
|
143
|
+
)
|
|
144
|
+
self._da = dataarray
|
|
145
|
+
self._driver = driver
|
|
146
|
+
self._source_path = source_path
|
|
147
|
+
|
|
148
|
+
# ========================
|
|
149
|
+
# Factories
|
|
150
|
+
# ========================
|
|
151
|
+
@classmethod
|
|
152
|
+
def from_path(cls, path: StrPath, *, chunks: ChunkSpec = "auto") -> XarrayAdapter:
|
|
153
|
+
"""Open a raster file lazily, split into dask chunks.
|
|
154
|
+
|
|
155
|
+
Parameters
|
|
156
|
+
----------
|
|
157
|
+
path : str or path-like
|
|
158
|
+
Path to a GDAL-readable raster.
|
|
159
|
+
chunks : str or int or dict, default "auto"
|
|
160
|
+
Chunk sizes, as :func:`rioxarray.open_rasterio` takes them.
|
|
161
|
+
``"auto"`` lets dask pick sizes aligned with the file's internal
|
|
162
|
+
blocks.
|
|
163
|
+
|
|
164
|
+
Returns
|
|
165
|
+
-------
|
|
166
|
+
XarrayAdapter
|
|
167
|
+
Adapter over the unread raster.
|
|
168
|
+
|
|
169
|
+
Raises
|
|
170
|
+
------
|
|
171
|
+
MissingDependencyError
|
|
172
|
+
If the ``lazy`` extra is not installed.
|
|
173
|
+
ValidationError
|
|
174
|
+
If ``chunks`` is not a valid chunk specification.
|
|
175
|
+
BackendError
|
|
176
|
+
If the file cannot be opened as a raster.
|
|
177
|
+
"""
|
|
178
|
+
# Widened because "auto" is resolved to concrete sizes below, and a
|
|
179
|
+
# dict[str, int] is not a dict[str, int | Literal["auto"]] to a type
|
|
180
|
+
# checker: dict values are invariant.
|
|
181
|
+
spec: ChunkSpec | dict[str, int] = validate_chunks(chunks)
|
|
182
|
+
rioxarray = _import_rioxarray()
|
|
183
|
+
# dask is not imported by rioxarray itself; check it up front so a
|
|
184
|
+
# missing package is reported as the extra, not as an xarray error.
|
|
185
|
+
import_optional("dask.array", extra="lazy", purpose=_PURPOSE)
|
|
186
|
+
try:
|
|
187
|
+
with open_env(path):
|
|
188
|
+
# rasterio supplies the driver, which rioxarray does not
|
|
189
|
+
# record, and the block layout "auto" is resolved against.
|
|
190
|
+
with rio.open(path) as src:
|
|
191
|
+
driver = src.driver
|
|
192
|
+
if spec == "auto":
|
|
193
|
+
spec = _auto_chunk_sizes(src)
|
|
194
|
+
dataarray = rioxarray.open_rasterio(path, chunks=spec)
|
|
195
|
+
except Exception as e:
|
|
196
|
+
raise BackendError(f"failed to open raster lazily: {path}") from e
|
|
197
|
+
return cls(dataarray, driver=driver, source_path=path)
|
|
198
|
+
|
|
199
|
+
@property
|
|
200
|
+
def source_path(self) -> StrPath | None:
|
|
201
|
+
"""File this array was opened from, or None if it was not opened from one."""
|
|
202
|
+
return self._source_path
|
|
203
|
+
|
|
204
|
+
# ========================
|
|
205
|
+
# Metadata
|
|
206
|
+
# ========================
|
|
207
|
+
def get_crs(self) -> CRS:
|
|
208
|
+
return self._da.rio.crs
|
|
209
|
+
|
|
210
|
+
def get_transform(self) -> Affine:
|
|
211
|
+
return self._da.rio.transform()
|
|
212
|
+
|
|
213
|
+
def get_bounds(self) -> BoundingBox:
|
|
214
|
+
height, width = self.get_shape()
|
|
215
|
+
return BoundingBox(*array_bounds(height, width, self.get_transform()))
|
|
216
|
+
|
|
217
|
+
def get_shape(self) -> tuple[int, int]:
|
|
218
|
+
return self.get_height(), self.get_width()
|
|
219
|
+
|
|
220
|
+
def get_width(self) -> int:
|
|
221
|
+
return int(self._da.sizes["x"])
|
|
222
|
+
|
|
223
|
+
def get_height(self) -> int:
|
|
224
|
+
return int(self._da.sizes["y"])
|
|
225
|
+
|
|
226
|
+
def get_count(self) -> int:
|
|
227
|
+
return int(self._da.sizes["band"])
|
|
228
|
+
|
|
229
|
+
def get_nodata(self) -> float | None:
|
|
230
|
+
# rioxarray reports nodata as a scalar of the band dtype; rasterio
|
|
231
|
+
# reports a float, and the rest of the library expects that.
|
|
232
|
+
nodata = self._da.rio.nodata
|
|
233
|
+
return None if nodata is None else float(nodata)
|
|
234
|
+
|
|
235
|
+
def get_metadata(self) -> dict:
|
|
236
|
+
return {
|
|
237
|
+
"driver": self._driver,
|
|
238
|
+
"dtype": str(self._da.dtype),
|
|
239
|
+
"nodata": self.get_nodata(),
|
|
240
|
+
"width": self.get_width(),
|
|
241
|
+
"height": self.get_height(),
|
|
242
|
+
"count": self.get_count(),
|
|
243
|
+
"crs": self.get_crs(),
|
|
244
|
+
"transform": self.get_transform(),
|
|
245
|
+
}
|
|
246
|
+
|
|
247
|
+
def get_band_descriptions(self) -> list[str | None]:
|
|
248
|
+
# rioxarray stores GDAL band descriptions as ``long_name``: a plain
|
|
249
|
+
# string for one band, a tuple otherwise, absent when none are set.
|
|
250
|
+
long_name = self._da.attrs.get("long_name")
|
|
251
|
+
if long_name is None:
|
|
252
|
+
return [None] * self.get_count()
|
|
253
|
+
if isinstance(long_name, str):
|
|
254
|
+
long_name = (long_name,)
|
|
255
|
+
return [(name or None) for name in long_name]
|
|
256
|
+
|
|
257
|
+
# ========================
|
|
258
|
+
# Data Access
|
|
259
|
+
# ========================
|
|
260
|
+
def read(
|
|
261
|
+
self,
|
|
262
|
+
indexes: int | Sequence[int] | None = None,
|
|
263
|
+
*,
|
|
264
|
+
window: Window | tuple | None = None,
|
|
265
|
+
**kwargs,
|
|
266
|
+
) -> np.ndarray:
|
|
267
|
+
"""Compute the requested bands and window as a NumPy array.
|
|
268
|
+
|
|
269
|
+
Follows :meth:`rasterio.io.DatasetReader.read` for the arguments it
|
|
270
|
+
supports: only the selected bands and window are computed, so reading
|
|
271
|
+
a window of a scene larger than memory stays bounded.
|
|
272
|
+
|
|
273
|
+
Parameters
|
|
274
|
+
----------
|
|
275
|
+
indexes : int or sequence of int or None, default None
|
|
276
|
+
1-based band index (returns a 2D array), a sequence of them, or
|
|
277
|
+
``None`` for every band (both return a 3D array).
|
|
278
|
+
window : rasterio.windows.Window or tuple or None, default None
|
|
279
|
+
Integer pixel window to read, or ``((row_start, row_stop),
|
|
280
|
+
(col_start, col_stop))``. ``None`` reads the whole raster.
|
|
281
|
+
**kwargs
|
|
282
|
+
Any other rasterio read option. None is supported.
|
|
283
|
+
|
|
284
|
+
Returns
|
|
285
|
+
-------
|
|
286
|
+
numpy.ndarray
|
|
287
|
+
The requested pixels, as stored.
|
|
288
|
+
|
|
289
|
+
Raises
|
|
290
|
+
------
|
|
291
|
+
BackendError
|
|
292
|
+
If a rasterio read option this backend does not support is given.
|
|
293
|
+
IndexError
|
|
294
|
+
If a band index is out of range.
|
|
295
|
+
ValidationError
|
|
296
|
+
If ``window`` has fractional offsets or sizes, or does not lie
|
|
297
|
+
within the raster.
|
|
298
|
+
"""
|
|
299
|
+
if kwargs:
|
|
300
|
+
raise BackendError(
|
|
301
|
+
f"the lazy backend's read() supports indexes and window only; "
|
|
302
|
+
f"got {', '.join(sorted(kwargs))}"
|
|
303
|
+
)
|
|
304
|
+
bands: int | list[int] | slice
|
|
305
|
+
if indexes is None:
|
|
306
|
+
bands = slice(None)
|
|
307
|
+
elif isinstance(indexes, (int, np.integer)):
|
|
308
|
+
bands = self._band_position(int(indexes))
|
|
309
|
+
else:
|
|
310
|
+
bands = [self._band_position(int(i)) for i in indexes]
|
|
311
|
+
|
|
312
|
+
rows, cols = self._window_slices(window)
|
|
313
|
+
return self._compute(self._da.data[bands, rows, cols])
|
|
314
|
+
|
|
315
|
+
def read_band(self, idx: int) -> np.ndarray:
|
|
316
|
+
return self.read(idx)
|
|
317
|
+
|
|
318
|
+
def _band_position(self, idx: int) -> int:
|
|
319
|
+
"""Map a 1-based band index to its 0-based array position."""
|
|
320
|
+
count = self.get_count()
|
|
321
|
+
if idx < 1 or idx > count:
|
|
322
|
+
raise IndexError(
|
|
323
|
+
f"band index {idx} out of range; dataset has {count} band(s) (valid 1..{count})"
|
|
324
|
+
)
|
|
325
|
+
return idx - 1
|
|
326
|
+
|
|
327
|
+
def _window_slices(self, window: Window | tuple | None) -> tuple[slice, slice]:
|
|
328
|
+
"""Validate a pixel window and return its row and column slices."""
|
|
329
|
+
if window is None:
|
|
330
|
+
return slice(None), slice(None)
|
|
331
|
+
win = window if isinstance(window, Window) else Window.from_slices(*window)
|
|
332
|
+
bounds = (win.row_off, win.col_off, win.height, win.width)
|
|
333
|
+
if any(float(v) != int(v) for v in bounds):
|
|
334
|
+
raise ValidationError(f"window must have integer offsets and sizes; got {win}")
|
|
335
|
+
row, col, height, width = (int(v) for v in bounds)
|
|
336
|
+
if row < 0 or col < 0 or row + height > self.get_height() or col + width > self.get_width():
|
|
337
|
+
raise ValidationError(
|
|
338
|
+
f"window {win} does not lie within the raster "
|
|
339
|
+
f"({self.get_height()} x {self.get_width()} pixels)"
|
|
340
|
+
)
|
|
341
|
+
return slice(row, row + height), slice(col, col + width)
|
|
342
|
+
|
|
343
|
+
@staticmethod
|
|
344
|
+
def _compute(data: Any) -> np.ndarray:
|
|
345
|
+
"""Materialise a selection as a NumPy array that owns its memory."""
|
|
346
|
+
if hasattr(data, "compute"):
|
|
347
|
+
return np.asarray(data.compute())
|
|
348
|
+
# An in-memory DataArray hands out views of its own buffer; copy, so a
|
|
349
|
+
# caller writing into the result cannot reach back into the dataset.
|
|
350
|
+
return np.array(data, copy=True)
|
|
351
|
+
|
|
352
|
+
# ========================
|
|
353
|
+
# Persistence
|
|
354
|
+
# ========================
|
|
355
|
+
def write(
|
|
356
|
+
self, path: StrPath, driver: str = "GTiff", band_names: list[str | None] | None = None
|
|
357
|
+
) -> None:
|
|
358
|
+
# One dask chunk is computed and written at a time, so peak memory
|
|
359
|
+
# follows the chunk size, not the raster size.
|
|
360
|
+
meta = self.get_metadata()
|
|
361
|
+
meta.update(driver=driver)
|
|
362
|
+
data = self._da.data
|
|
363
|
+
chunks = getattr(data, "chunks", None) or tuple((n,) for n in data.shape)
|
|
364
|
+
|
|
365
|
+
with rio.open(path, "w", **meta) as dst:
|
|
366
|
+
for band_s, row_s, col_s in itertools.product(*(_spans(c) for c in chunks)):
|
|
367
|
+
block = self._compute(data[band_s, row_s, col_s])
|
|
368
|
+
dst.write(
|
|
369
|
+
block,
|
|
370
|
+
indexes=list(range(band_s.start + 1, band_s.stop + 1)),
|
|
371
|
+
window=Window.from_slices(row_s, col_s),
|
|
372
|
+
)
|
|
373
|
+
# Unnamed bands are left alone so the file records no description.
|
|
374
|
+
for i, name in enumerate(band_names or [], start=1):
|
|
375
|
+
if name:
|
|
376
|
+
dst.set_band_description(i, name)
|
|
377
|
+
|
|
378
|
+
def close(self) -> None:
|
|
379
|
+
self._da.close()
|
|
380
|
+
|
|
381
|
+
# ========================
|
|
382
|
+
# Backend Access
|
|
383
|
+
# ========================
|
|
384
|
+
@property
|
|
385
|
+
def backend(self) -> Any:
|
|
386
|
+
return self._da
|
|
387
|
+
|
|
388
|
+
|
|
389
|
+
def _spans(sizes: Sequence[int]) -> list[slice]:
|
|
390
|
+
"""Turn one axis's chunk sizes into consecutive slices."""
|
|
391
|
+
stops = np.cumsum(sizes).tolist()
|
|
392
|
+
starts = [0, *stops[:-1]]
|
|
393
|
+
return [slice(start, stop) for start, stop in zip(starts, stops, strict=True)]
|
|
@@ -25,6 +25,7 @@ from rasterio.windows import Window
|
|
|
25
25
|
from eeo.common import (
|
|
26
26
|
apply_nodata_mask,
|
|
27
27
|
get_nodata,
|
|
28
|
+
resolve_band_index,
|
|
28
29
|
resolve_output_dtype,
|
|
29
30
|
resolve_output_nodata,
|
|
30
31
|
)
|
|
@@ -86,7 +87,7 @@ class BlockSource:
|
|
|
86
87
|
self.nodata = nodata
|
|
87
88
|
|
|
88
89
|
@classmethod
|
|
89
|
-
def from_dataset(cls, ds: EEORasterDataset, *, band: int | None = None) -> BlockSource:
|
|
90
|
+
def from_dataset(cls, ds: EEORasterDataset, *, band: int | str | None = None) -> BlockSource:
|
|
90
91
|
"""Wrap a raster operand, promoting it to the rasterio backend.
|
|
91
92
|
|
|
92
93
|
Promotion matters for correctness, not just speed: the NumPy adapter's
|
|
@@ -97,15 +98,25 @@ class BlockSource:
|
|
|
97
98
|
----------
|
|
98
99
|
ds : EEORasterDataset
|
|
99
100
|
Raster operand.
|
|
100
|
-
band : int or None, default None
|
|
101
|
-
1-based band index to read, or None to read every
|
|
101
|
+
band : int or str or None, default None
|
|
102
|
+
1-based band index or band name to read, or None to read every
|
|
103
|
+
band. A name is resolved as it is everywhere else a band is
|
|
104
|
+
addressed.
|
|
102
105
|
|
|
103
106
|
Returns
|
|
104
107
|
-------
|
|
105
108
|
BlockSource
|
|
106
109
|
Source reading ``ds`` one window at a time.
|
|
110
|
+
|
|
111
|
+
Raises
|
|
112
|
+
------
|
|
113
|
+
IndexError
|
|
114
|
+
If an int index is outside the range of available bands.
|
|
115
|
+
ValidationError
|
|
116
|
+
If a name is unknown or declared on more than one band.
|
|
107
117
|
"""
|
|
108
|
-
|
|
118
|
+
index = None if band is None else resolve_band_index(ds, band)
|
|
119
|
+
return cls(dataset=ds.to_rasterio(), band=index, value=None, nodata=get_nodata(ds))
|
|
109
120
|
|
|
110
121
|
@classmethod
|
|
111
122
|
def from_scalar(cls, value: float) -> BlockSource:
|