mapchete-eo 2026.2.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- mapchete_eo/__init__.py +1 -0
- mapchete_eo/array/__init__.py +0 -0
- mapchete_eo/array/buffer.py +16 -0
- mapchete_eo/array/color.py +29 -0
- mapchete_eo/array/convert.py +163 -0
- mapchete_eo/base.py +653 -0
- mapchete_eo/blacklist.txt +175 -0
- mapchete_eo/cli/__init__.py +30 -0
- mapchete_eo/cli/bounds.py +22 -0
- mapchete_eo/cli/options_arguments.py +227 -0
- mapchete_eo/cli/s2_brdf.py +77 -0
- mapchete_eo/cli/s2_cat_results.py +130 -0
- mapchete_eo/cli/s2_find_broken_products.py +77 -0
- mapchete_eo/cli/s2_jp2_static_catalog.py +166 -0
- mapchete_eo/cli/s2_mask.py +71 -0
- mapchete_eo/cli/s2_mgrs.py +45 -0
- mapchete_eo/cli/s2_rgb.py +114 -0
- mapchete_eo/cli/s2_verify.py +129 -0
- mapchete_eo/cli/static_catalog.py +82 -0
- mapchete_eo/eostac.py +30 -0
- mapchete_eo/exceptions.py +87 -0
- mapchete_eo/image_operations/__init__.py +12 -0
- mapchete_eo/image_operations/blend_functions.py +579 -0
- mapchete_eo/image_operations/color_correction.py +136 -0
- mapchete_eo/image_operations/compositing.py +266 -0
- mapchete_eo/image_operations/dtype_scale.py +43 -0
- mapchete_eo/image_operations/fillnodata.py +130 -0
- mapchete_eo/image_operations/filters.py +319 -0
- mapchete_eo/image_operations/linear_normalization.py +81 -0
- mapchete_eo/image_operations/sigmoidal.py +114 -0
- mapchete_eo/io/__init__.py +37 -0
- mapchete_eo/io/assets.py +496 -0
- mapchete_eo/io/items.py +162 -0
- mapchete_eo/io/levelled_cubes.py +259 -0
- mapchete_eo/io/path.py +155 -0
- mapchete_eo/io/products.py +423 -0
- mapchete_eo/io/profiles.py +45 -0
- mapchete_eo/platforms/sentinel2/__init__.py +17 -0
- mapchete_eo/platforms/sentinel2/_mapper_registry.py +89 -0
- mapchete_eo/platforms/sentinel2/bandpass_adjustment.py +104 -0
- mapchete_eo/platforms/sentinel2/brdf/__init__.py +8 -0
- mapchete_eo/platforms/sentinel2/brdf/config.py +32 -0
- mapchete_eo/platforms/sentinel2/brdf/correction.py +260 -0
- mapchete_eo/platforms/sentinel2/brdf/hls.py +251 -0
- mapchete_eo/platforms/sentinel2/brdf/models.py +44 -0
- mapchete_eo/platforms/sentinel2/brdf/protocols.py +27 -0
- mapchete_eo/platforms/sentinel2/brdf/ross_thick.py +136 -0
- mapchete_eo/platforms/sentinel2/brdf/sun_angle_arrays.py +76 -0
- mapchete_eo/platforms/sentinel2/config.py +241 -0
- mapchete_eo/platforms/sentinel2/driver.py +43 -0
- mapchete_eo/platforms/sentinel2/masks.py +329 -0
- mapchete_eo/platforms/sentinel2/metadata_parser/__init__.py +6 -0
- mapchete_eo/platforms/sentinel2/metadata_parser/base.py +56 -0
- mapchete_eo/platforms/sentinel2/metadata_parser/default_path_mapper.py +135 -0
- mapchete_eo/platforms/sentinel2/metadata_parser/models.py +78 -0
- mapchete_eo/platforms/sentinel2/metadata_parser/s2metadata.py +639 -0
- mapchete_eo/platforms/sentinel2/preconfigured_sources/__init__.py +57 -0
- mapchete_eo/platforms/sentinel2/preconfigured_sources/guessers.py +108 -0
- mapchete_eo/platforms/sentinel2/preconfigured_sources/item_mappers.py +171 -0
- mapchete_eo/platforms/sentinel2/preconfigured_sources/metadata_xml_mappers.py +217 -0
- mapchete_eo/platforms/sentinel2/preprocessing_tasks.py +50 -0
- mapchete_eo/platforms/sentinel2/processing_baseline.py +163 -0
- mapchete_eo/platforms/sentinel2/product.py +747 -0
- mapchete_eo/platforms/sentinel2/source.py +114 -0
- mapchete_eo/platforms/sentinel2/types.py +114 -0
- mapchete_eo/processes/__init__.py +0 -0
- mapchete_eo/processes/config.py +51 -0
- mapchete_eo/processes/dtype_scale.py +112 -0
- mapchete_eo/processes/eo_to_xarray.py +19 -0
- mapchete_eo/processes/merge_rasters.py +239 -0
- mapchete_eo/product.py +323 -0
- mapchete_eo/protocols.py +61 -0
- mapchete_eo/search/__init__.py +14 -0
- mapchete_eo/search/base.py +285 -0
- mapchete_eo/search/config.py +113 -0
- mapchete_eo/search/s2_mgrs.py +313 -0
- mapchete_eo/search/stac_search.py +278 -0
- mapchete_eo/search/stac_static.py +197 -0
- mapchete_eo/search/utm_search.py +251 -0
- mapchete_eo/settings.py +25 -0
- mapchete_eo/sort.py +60 -0
- mapchete_eo/source.py +109 -0
- mapchete_eo/time.py +62 -0
- mapchete_eo/types.py +76 -0
- mapchete_eo-2026.2.0.dist-info/METADATA +91 -0
- mapchete_eo-2026.2.0.dist-info/RECORD +89 -0
- mapchete_eo-2026.2.0.dist-info/WHEEL +4 -0
- mapchete_eo-2026.2.0.dist-info/entry_points.txt +11 -0
- mapchete_eo-2026.2.0.dist-info/licenses/LICENSE +21 -0
|
@@ -0,0 +1,747 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
import logging
|
|
4
|
+
from typing import Any, Callable, Dict, List, Optional, Union
|
|
5
|
+
|
|
6
|
+
import numpy as np
|
|
7
|
+
import numpy.ma as ma
|
|
8
|
+
from mapchete.io.raster import ReferencedRaster, read_raster_window, resample_from_array
|
|
9
|
+
from mapchete.geometry import reproject_geometry, buffer_antimeridian_safe
|
|
10
|
+
from mapchete.path import MPath
|
|
11
|
+
from mapchete.protocols import GridProtocol
|
|
12
|
+
from mapchete.types import Bounds, Grid, NodataVals
|
|
13
|
+
from pystac import Item
|
|
14
|
+
from rasterio.enums import Resampling
|
|
15
|
+
from rasterio.features import rasterize
|
|
16
|
+
from shapely.geometry import shape
|
|
17
|
+
|
|
18
|
+
|
|
19
|
+
from mapchete_eo.array.buffer import buffer_array
|
|
20
|
+
from mapchete_eo.io.items import get_item_property
|
|
21
|
+
from mapchete_eo.platforms.sentinel2.brdf.config import BRDFModels
|
|
22
|
+
from mapchete_eo.platforms.sentinel2.brdf.correction import apply_correction
|
|
23
|
+
from mapchete_eo.exceptions import (
|
|
24
|
+
AllMasked,
|
|
25
|
+
AssetError,
|
|
26
|
+
BRDFError,
|
|
27
|
+
CorruptedProduct,
|
|
28
|
+
EmptyFootprintException,
|
|
29
|
+
EmptyProductException,
|
|
30
|
+
)
|
|
31
|
+
from mapchete_eo.io.assets import get_assets, read_mask_as_raster
|
|
32
|
+
from mapchete_eo.io.path import asset_mpath, get_product_cache_path
|
|
33
|
+
from mapchete_eo.io.profiles import COGDeflateProfile
|
|
34
|
+
from mapchete_eo.platforms.sentinel2.brdf import correction_values
|
|
35
|
+
from mapchete_eo.platforms.sentinel2.bandpass_adjustment import (
|
|
36
|
+
apply_bandpass_adjustment,
|
|
37
|
+
)
|
|
38
|
+
from mapchete_eo.platforms.sentinel2.config import (
|
|
39
|
+
BRDFConfig,
|
|
40
|
+
BRDFModelConfig,
|
|
41
|
+
CacheConfig,
|
|
42
|
+
MaskConfig,
|
|
43
|
+
)
|
|
44
|
+
from mapchete_eo.platforms.sentinel2.metadata_parser.s2metadata import S2Metadata
|
|
45
|
+
from mapchete_eo.platforms.sentinel2.types import (
|
|
46
|
+
CloudType,
|
|
47
|
+
L2ABand,
|
|
48
|
+
ProductQIMaskResolution,
|
|
49
|
+
Resolution,
|
|
50
|
+
)
|
|
51
|
+
from mapchete_eo.product import EOProduct, add_to_blacklist
|
|
52
|
+
from mapchete_eo.protocols import EOProductProtocol
|
|
53
|
+
from mapchete_eo.settings import mapchete_eo_settings
|
|
54
|
+
|
|
55
|
+
logger = logging.getLogger(__name__)
|
|
56
|
+
|
|
57
|
+
|
|
58
|
+
class Cache:
|
|
59
|
+
item: Item
|
|
60
|
+
config: CacheConfig
|
|
61
|
+
path: MPath
|
|
62
|
+
|
|
63
|
+
def __init__(self, item: Item, config: CacheConfig):
|
|
64
|
+
self.item = item
|
|
65
|
+
self.config = config
|
|
66
|
+
# TODO: maybe move this function here
|
|
67
|
+
self.path = get_product_cache_path(
|
|
68
|
+
self.item,
|
|
69
|
+
MPath.from_inp(self.config.path),
|
|
70
|
+
self.config.product_path_generation_method,
|
|
71
|
+
)
|
|
72
|
+
self.path.makedirs()
|
|
73
|
+
self._brdf_grid_cache: dict = dict()
|
|
74
|
+
if self.config.brdf:
|
|
75
|
+
self._brdf_bands = [
|
|
76
|
+
asset_name_to_l2a_band(self.item, band)
|
|
77
|
+
for band in self.config.brdf.bands
|
|
78
|
+
]
|
|
79
|
+
else:
|
|
80
|
+
self._brdf_bands = []
|
|
81
|
+
try:
|
|
82
|
+
self._existing_files = self.path.ls()
|
|
83
|
+
except FileNotFoundError:
|
|
84
|
+
self._existing_files = None
|
|
85
|
+
|
|
86
|
+
def __repr__(self):
|
|
87
|
+
return f"<Cache: product={self.item.id}, path={self.path}>"
|
|
88
|
+
|
|
89
|
+
def cache_assets(self):
|
|
90
|
+
# cache assets
|
|
91
|
+
if self.config.assets:
|
|
92
|
+
# TODO determine already existing assets
|
|
93
|
+
self.item = get_assets(
|
|
94
|
+
self.item,
|
|
95
|
+
self.config.assets,
|
|
96
|
+
self.path,
|
|
97
|
+
resolution=self.config.assets_resolution.value,
|
|
98
|
+
ignore_if_exists=True,
|
|
99
|
+
item_href_in_dst_dir=False,
|
|
100
|
+
)
|
|
101
|
+
return self.item
|
|
102
|
+
|
|
103
|
+
def cache_brdf_grids(self, metadata: S2Metadata):
|
|
104
|
+
if self.config.brdf:
|
|
105
|
+
resolution = self.config.brdf.resolution
|
|
106
|
+
model = self.config.brdf.model
|
|
107
|
+
|
|
108
|
+
logger.debug(
|
|
109
|
+
f"prepare BRDF model '{model}' for product bands {self._brdf_bands} in {resolution} resolution"
|
|
110
|
+
)
|
|
111
|
+
for band in self._brdf_bands:
|
|
112
|
+
out_path = self.path / f"brdf_{model}_{band.name}_{resolution}.tif"
|
|
113
|
+
# TODO: do check with _existing_files again to reduce S3 requests
|
|
114
|
+
if not out_path.exists():
|
|
115
|
+
try:
|
|
116
|
+
grid = correction_values(
|
|
117
|
+
metadata,
|
|
118
|
+
band,
|
|
119
|
+
model=model,
|
|
120
|
+
resolution=resolution,
|
|
121
|
+
per_detector=self.config.brdf.per_detector_correction,
|
|
122
|
+
)
|
|
123
|
+
except BRDFError as exc:
|
|
124
|
+
error_msg = (
|
|
125
|
+
f"product {self.item.get_self_href()} is corrupted: {exc}"
|
|
126
|
+
)
|
|
127
|
+
logger.error(error_msg)
|
|
128
|
+
add_to_blacklist(self.item.get_self_href())
|
|
129
|
+
raise CorruptedProduct(error_msg)
|
|
130
|
+
|
|
131
|
+
logger.debug("cache BRDF correction grid to %s", out_path)
|
|
132
|
+
grid.to_file(out_path, **COGDeflateProfile(grid.meta))
|
|
133
|
+
self._brdf_grid_cache[band] = out_path
|
|
134
|
+
|
|
135
|
+
def get_brdf_grid(self, band: L2ABand):
|
|
136
|
+
try:
|
|
137
|
+
return self._brdf_grid_cache[band]
|
|
138
|
+
except KeyError:
|
|
139
|
+
if band in self._brdf_bands:
|
|
140
|
+
raise KeyError(f"BRDF grid for band {band} not yet cached")
|
|
141
|
+
else:
|
|
142
|
+
raise KeyError(f"BRDF grid for band {band} not configured")
|
|
143
|
+
|
|
144
|
+
|
|
145
|
+
class S2Product(EOProduct, EOProductProtocol):
|
|
146
|
+
"""
|
|
147
|
+
Sentinel-2 specific EOProduct implementation.
|
|
148
|
+
"""
|
|
149
|
+
|
|
150
|
+
_item_dict: Optional[dict] = None
|
|
151
|
+
cache: Optional[Cache] = None
|
|
152
|
+
_scl_cache: Dict[GridProtocol, np.ndarray]
|
|
153
|
+
_item_property_cache: Dict[str, Any]
|
|
154
|
+
|
|
155
|
+
def __init__(
|
|
156
|
+
self,
|
|
157
|
+
item: Item,
|
|
158
|
+
metadata: Optional[S2Metadata] = None,
|
|
159
|
+
cache_config: Optional[CacheConfig] = None,
|
|
160
|
+
metadata_mapper: Optional[Callable[[Item], S2Metadata]] = None,
|
|
161
|
+
item_modifier_funcs: Optional[List[Callable[[Item], Item]]] = None,
|
|
162
|
+
lazy_load_item: bool = False,
|
|
163
|
+
item_property_cache: Optional[Dict[str, Any]] = None,
|
|
164
|
+
):
|
|
165
|
+
if lazy_load_item:
|
|
166
|
+
self._item_dict = None
|
|
167
|
+
else:
|
|
168
|
+
self._item_dict = item.to_dict()
|
|
169
|
+
self.item_href = item.self_href
|
|
170
|
+
self.id = item.id
|
|
171
|
+
|
|
172
|
+
self._metadata = metadata
|
|
173
|
+
self._metadata_mapper = metadata_mapper
|
|
174
|
+
self._item_modifier_funcs = item_modifier_funcs
|
|
175
|
+
self._scl_cache = dict()
|
|
176
|
+
self._item_property_cache = item_property_cache or dict()
|
|
177
|
+
self.cache = Cache(item, cache_config) if cache_config else None
|
|
178
|
+
|
|
179
|
+
self.__geo_interface__ = item.geometry
|
|
180
|
+
self.bounds = Bounds.from_inp(shape(self))
|
|
181
|
+
self.crs = mapchete_eo_settings.default_catalog_crs
|
|
182
|
+
|
|
183
|
+
@classmethod
|
|
184
|
+
def from_stac_item(
|
|
185
|
+
self,
|
|
186
|
+
item: Item,
|
|
187
|
+
cache_config: Optional[CacheConfig] = None,
|
|
188
|
+
cache_all: bool = False,
|
|
189
|
+
**kwargs,
|
|
190
|
+
) -> S2Product:
|
|
191
|
+
s2product = S2Product(item, cache_config=cache_config, **kwargs)
|
|
192
|
+
|
|
193
|
+
if cache_all:
|
|
194
|
+
# cache assets if configured
|
|
195
|
+
s2product.cache_assets()
|
|
196
|
+
|
|
197
|
+
# cache BRDF grids if configured
|
|
198
|
+
s2product.cache_brdf_grids()
|
|
199
|
+
|
|
200
|
+
return s2product
|
|
201
|
+
|
|
202
|
+
@property
|
|
203
|
+
def item(self) -> Item:
|
|
204
|
+
if not self._item:
|
|
205
|
+
if self._item_dict:
|
|
206
|
+
self._item = Item.from_dict(self._item_dict)
|
|
207
|
+
else:
|
|
208
|
+
item = Item.from_file(self.item_href)
|
|
209
|
+
for modifier in self._item_modifier_funcs or []:
|
|
210
|
+
item = modifier(item)
|
|
211
|
+
self._item = item
|
|
212
|
+
return self._item
|
|
213
|
+
|
|
214
|
+
@property
|
|
215
|
+
def metadata(self) -> S2Metadata:
|
|
216
|
+
if not self._metadata:
|
|
217
|
+
if self._metadata_mapper:
|
|
218
|
+
self._metadata = self._metadata_mapper(self.item)
|
|
219
|
+
else:
|
|
220
|
+
self._metadata = S2Metadata.from_stac_item(self.item)
|
|
221
|
+
return self._metadata
|
|
222
|
+
|
|
223
|
+
def __repr__(self):
|
|
224
|
+
return f"<S2Product product_id={self.id}>"
|
|
225
|
+
|
|
226
|
+
def clear_cached_data(self):
|
|
227
|
+
if self._metadata is not None:
|
|
228
|
+
self._metadata.clear_cached_data()
|
|
229
|
+
self._metadata = None
|
|
230
|
+
if self._item is not None:
|
|
231
|
+
self._item = None
|
|
232
|
+
self._item_property_cache = dict()
|
|
233
|
+
self._scl_cache = dict()
|
|
234
|
+
|
|
235
|
+
def read_np_array(
|
|
236
|
+
self,
|
|
237
|
+
assets: Optional[List[str]] = None,
|
|
238
|
+
eo_bands: Optional[List[str]] = None,
|
|
239
|
+
grid: Union[GridProtocol, Resolution] = Resolution["10m"],
|
|
240
|
+
resampling: Resampling = Resampling.nearest,
|
|
241
|
+
nodatavals: NodataVals = None,
|
|
242
|
+
raise_empty: bool = True,
|
|
243
|
+
apply_offset: bool = True,
|
|
244
|
+
apply_scale: bool = False,
|
|
245
|
+
apply_sentinel2_bandpass_adjustment: bool = False,
|
|
246
|
+
mask_config: MaskConfig = MaskConfig(),
|
|
247
|
+
brdf_config: Optional[BRDFConfig] = None,
|
|
248
|
+
fill_value: int = 0,
|
|
249
|
+
read_mask: Optional[np.ndarray] = None,
|
|
250
|
+
**kwargs,
|
|
251
|
+
) -> ma.MaskedArray:
|
|
252
|
+
"""
|
|
253
|
+
Read Sentinel-2 assets into a MaskedArray with masks and BRDF.
|
|
254
|
+
"""
|
|
255
|
+
assets = assets or []
|
|
256
|
+
eo_bands = eo_bands or []
|
|
257
|
+
apply_offset = apply_offset and not self.metadata.boa_offset_applied
|
|
258
|
+
if eo_bands:
|
|
259
|
+
count = len(eo_bands)
|
|
260
|
+
raise NotImplementedError("please use asset names for now")
|
|
261
|
+
else:
|
|
262
|
+
count = len(assets)
|
|
263
|
+
if isinstance(grid, Resolution):
|
|
264
|
+
grid = self.metadata.grid(grid)
|
|
265
|
+
mask = self.get_mask(
|
|
266
|
+
grid, mask_config, target_mask=None if read_mask is None else ~read_mask
|
|
267
|
+
).data
|
|
268
|
+
if nodatavals is None:
|
|
269
|
+
nodatavals = fill_value
|
|
270
|
+
elif fill_value is None and nodatavals is not None:
|
|
271
|
+
fill_value = nodatavals
|
|
272
|
+
if mask.all():
|
|
273
|
+
if raise_empty:
|
|
274
|
+
raise EmptyProductException(
|
|
275
|
+
f"{self}: configured mask over {grid} covers everything"
|
|
276
|
+
)
|
|
277
|
+
else:
|
|
278
|
+
return self.empty_array(count, grid=grid, fill_value=fill_value)
|
|
279
|
+
|
|
280
|
+
arr = super().read_np_array(
|
|
281
|
+
assets=assets,
|
|
282
|
+
eo_bands=eo_bands,
|
|
283
|
+
grid=grid,
|
|
284
|
+
resampling=resampling,
|
|
285
|
+
raise_empty=False,
|
|
286
|
+
apply_offset=apply_offset,
|
|
287
|
+
apply_scale=apply_scale,
|
|
288
|
+
)
|
|
289
|
+
|
|
290
|
+
# bring mask to same shape as data array
|
|
291
|
+
expanded_mask = np.repeat(np.expand_dims(mask, axis=0), arr.shape[0], axis=0)
|
|
292
|
+
arr.set_fill_value(fill_value)
|
|
293
|
+
arr[expanded_mask] = fill_value
|
|
294
|
+
arr[expanded_mask] = ma.masked
|
|
295
|
+
|
|
296
|
+
if arr.mask.all():
|
|
297
|
+
if raise_empty:
|
|
298
|
+
raise EmptyProductException(
|
|
299
|
+
f"{self}: is empty over {grid} after reading bands and applying all masks"
|
|
300
|
+
)
|
|
301
|
+
else:
|
|
302
|
+
return self.empty_array(count, grid=grid, fill_value=fill_value)
|
|
303
|
+
|
|
304
|
+
# apply Sentinel-2 bandpass adjustment
|
|
305
|
+
if apply_sentinel2_bandpass_adjustment:
|
|
306
|
+
arr = self._apply_sentinel2_bandpass_adjustment(
|
|
307
|
+
uncorrected=arr, assets=assets
|
|
308
|
+
)
|
|
309
|
+
|
|
310
|
+
# apply BRDF config if required
|
|
311
|
+
if brdf_config:
|
|
312
|
+
arr = self._apply_brdf(
|
|
313
|
+
uncorrected=arr,
|
|
314
|
+
assets=assets,
|
|
315
|
+
brdf_config=brdf_config,
|
|
316
|
+
grid=grid,
|
|
317
|
+
resampling=resampling,
|
|
318
|
+
mask_config=mask_config,
|
|
319
|
+
)
|
|
320
|
+
|
|
321
|
+
return ma.MaskedArray(arr, fill_value=fill_value)
|
|
322
|
+
|
|
323
|
+
def cache_assets(self) -> None:
|
|
324
|
+
if self.cache is not None:
|
|
325
|
+
self.cache.cache_assets()
|
|
326
|
+
|
|
327
|
+
def cache_brdf_grids(self) -> None:
|
|
328
|
+
if self.cache is not None:
|
|
329
|
+
self.cache.cache_brdf_grids(self.metadata)
|
|
330
|
+
|
|
331
|
+
def read_brdf_grid(
|
|
332
|
+
self,
|
|
333
|
+
band: L2ABand,
|
|
334
|
+
resampling: Resampling = Resampling.bilinear,
|
|
335
|
+
grid: Union[GridProtocol, Resolution] = Resolution["20m"],
|
|
336
|
+
brdf_config: BRDFModelConfig = BRDFConfig(),
|
|
337
|
+
) -> np.ndarray:
|
|
338
|
+
grid = (
|
|
339
|
+
self.metadata.grid(grid)
|
|
340
|
+
if isinstance(grid, Resolution)
|
|
341
|
+
else Grid.from_obj(grid)
|
|
342
|
+
)
|
|
343
|
+
try:
|
|
344
|
+
# read cached file if configured
|
|
345
|
+
if self.cache:
|
|
346
|
+
return read_raster_window(
|
|
347
|
+
self.cache.get_brdf_grid(band),
|
|
348
|
+
grid=grid,
|
|
349
|
+
resampling=resampling,
|
|
350
|
+
)
|
|
351
|
+
# calculate on the fly
|
|
352
|
+
return resample_from_array(
|
|
353
|
+
correction_values(
|
|
354
|
+
self.metadata,
|
|
355
|
+
band,
|
|
356
|
+
model=brdf_config.model,
|
|
357
|
+
resolution=brdf_config.resolution,
|
|
358
|
+
footprints_cached_read=brdf_config.footprints_cached_read,
|
|
359
|
+
per_detector=brdf_config.per_detector_correction,
|
|
360
|
+
),
|
|
361
|
+
out_grid=grid,
|
|
362
|
+
resampling=resampling,
|
|
363
|
+
keep_2d=True,
|
|
364
|
+
)
|
|
365
|
+
except (AssetError, BRDFError) as exc:
|
|
366
|
+
error_msg = f"product {self.item.get_self_href()} is corrupted: {exc}"
|
|
367
|
+
logger.error(error_msg)
|
|
368
|
+
add_to_blacklist(self.item.get_self_href())
|
|
369
|
+
raise CorruptedProduct(error_msg)
|
|
370
|
+
|
|
371
|
+
def read_l1c_cloud_mask(
|
|
372
|
+
self,
|
|
373
|
+
grid: Union[GridProtocol, Resolution] = Resolution["20m"],
|
|
374
|
+
cloud_type: CloudType = CloudType.all,
|
|
375
|
+
cached_read: bool = False,
|
|
376
|
+
) -> ReferencedRaster:
|
|
377
|
+
"""Return classification cloud mask."""
|
|
378
|
+
logger.debug("read classification cloud mask for %s", str(self))
|
|
379
|
+
return self.metadata.l1c_cloud_mask(
|
|
380
|
+
cloud_type, dst_grid=grid, cached_read=cached_read
|
|
381
|
+
)
|
|
382
|
+
|
|
383
|
+
def read_snow_ice_mask(
|
|
384
|
+
self,
|
|
385
|
+
grid: Union[GridProtocol, Resolution] = Resolution["20m"],
|
|
386
|
+
cached_read: bool = False,
|
|
387
|
+
) -> ReferencedRaster:
|
|
388
|
+
"""Return classification snow and ice mask."""
|
|
389
|
+
logger.debug("read classification snow and ice mask for %s", str(self))
|
|
390
|
+
return self.metadata.snow_ice_mask(dst_grid=grid, cached_read=cached_read)
|
|
391
|
+
|
|
392
|
+
def read_cloud_probability(
|
|
393
|
+
self,
|
|
394
|
+
grid: Union[GridProtocol, Resolution] = Resolution["20m"],
|
|
395
|
+
resampling: Resampling = Resampling.bilinear,
|
|
396
|
+
from_resolution: ProductQIMaskResolution = ProductQIMaskResolution["20m"],
|
|
397
|
+
cached_read: bool = False,
|
|
398
|
+
) -> ReferencedRaster:
|
|
399
|
+
"""Return cloud probability mask."""
|
|
400
|
+
if "cloud" in self.item.assets:
|
|
401
|
+
logger.debug("read cloud probability mask for %s from asset", str(self))
|
|
402
|
+
return read_mask_as_raster(
|
|
403
|
+
path=asset_mpath(item=self.item, asset="cloud"),
|
|
404
|
+
dst_grid=(
|
|
405
|
+
self.metadata.grid(grid)
|
|
406
|
+
if isinstance(grid, Resolution)
|
|
407
|
+
else Grid.from_obj(grid)
|
|
408
|
+
),
|
|
409
|
+
resampling=resampling,
|
|
410
|
+
rasterize_value_func=lambda feature: True,
|
|
411
|
+
masked=False,
|
|
412
|
+
cached_read=cached_read,
|
|
413
|
+
)
|
|
414
|
+
logger.debug(
|
|
415
|
+
"read cloud probability mask for %s from metadata archive", str(self)
|
|
416
|
+
)
|
|
417
|
+
return self.metadata.cloud_probability(
|
|
418
|
+
dst_grid=grid,
|
|
419
|
+
resampling=resampling,
|
|
420
|
+
from_resolution=from_resolution,
|
|
421
|
+
cached_read=cached_read,
|
|
422
|
+
)
|
|
423
|
+
|
|
424
|
+
def read_snow_probability(
|
|
425
|
+
self,
|
|
426
|
+
grid: Union[GridProtocol, Resolution] = Resolution["20m"],
|
|
427
|
+
resampling: Resampling = Resampling.bilinear,
|
|
428
|
+
from_resolution: ProductQIMaskResolution = ProductQIMaskResolution["20m"],
|
|
429
|
+
cached_read: bool = False,
|
|
430
|
+
) -> ReferencedRaster:
|
|
431
|
+
"""Return classification snow and ice mask."""
|
|
432
|
+
if "snow" in self.item.assets:
|
|
433
|
+
logger.debug("read snow probability mask for %s from asset", str(self))
|
|
434
|
+
return read_mask_as_raster(
|
|
435
|
+
path=asset_mpath(item=self.item, asset="cloud"),
|
|
436
|
+
dst_grid=(
|
|
437
|
+
self.metadata.grid(grid)
|
|
438
|
+
if isinstance(grid, Resolution)
|
|
439
|
+
else Grid.from_obj(grid)
|
|
440
|
+
),
|
|
441
|
+
resampling=resampling,
|
|
442
|
+
rasterize_value_func=lambda feature: True,
|
|
443
|
+
masked=False,
|
|
444
|
+
cached_read=cached_read,
|
|
445
|
+
)
|
|
446
|
+
logger.debug(
|
|
447
|
+
"read snow probability mask for %s from metadata archive", str(self)
|
|
448
|
+
)
|
|
449
|
+
return self.metadata.snow_probability(
|
|
450
|
+
dst_grid=grid,
|
|
451
|
+
resampling=resampling,
|
|
452
|
+
from_resolution=from_resolution,
|
|
453
|
+
cached_read=cached_read,
|
|
454
|
+
)
|
|
455
|
+
|
|
456
|
+
def read_scl(
|
|
457
|
+
self,
|
|
458
|
+
grid: Union[GridProtocol, Resolution] = Resolution["20m"],
|
|
459
|
+
cached_read: bool = False,
|
|
460
|
+
) -> ReferencedRaster:
|
|
461
|
+
"""
|
|
462
|
+
Read Scene Classification Layer mask.
|
|
463
|
+
"""
|
|
464
|
+
grid = (
|
|
465
|
+
self.metadata.grid(grid)
|
|
466
|
+
if isinstance(grid, Resolution)
|
|
467
|
+
else Grid.from_obj(grid)
|
|
468
|
+
)
|
|
469
|
+
grid_hash = hash((grid.transform, grid.shape))
|
|
470
|
+
if grid_hash not in self._scl_cache:
|
|
471
|
+
logger.debug("read SCL mask for %s", str(self))
|
|
472
|
+
self._scl_cache[grid_hash] = read_mask_as_raster(
|
|
473
|
+
asset_mpath(self.item, "scl"),
|
|
474
|
+
dst_grid=grid,
|
|
475
|
+
resampling=Resampling.nearest,
|
|
476
|
+
masked=True,
|
|
477
|
+
cached_read=cached_read,
|
|
478
|
+
)
|
|
479
|
+
return self._scl_cache[grid_hash]
|
|
480
|
+
|
|
481
|
+
def footprint_nodata_mask(
|
|
482
|
+
self,
|
|
483
|
+
grid: Union[GridProtocol, Resolution] = Resolution["10m"],
|
|
484
|
+
buffer_m: float = 0,
|
|
485
|
+
) -> ReferencedRaster:
|
|
486
|
+
"""Return rasterized footprint mask."""
|
|
487
|
+
grid = (
|
|
488
|
+
self.metadata.grid(grid)
|
|
489
|
+
if isinstance(grid, Resolution)
|
|
490
|
+
else Grid.from_obj(grid)
|
|
491
|
+
)
|
|
492
|
+
if buffer_m:
|
|
493
|
+
footprint = buffer_antimeridian_safe(shape(self), buffer_m=buffer_m)
|
|
494
|
+
if footprint.is_empty:
|
|
495
|
+
raise EmptyFootprintException(
|
|
496
|
+
f"buffer value of {buffer_m} results in an empty geometry for footprint {shape(self).wkt}"
|
|
497
|
+
)
|
|
498
|
+
else:
|
|
499
|
+
footprint = shape(self)
|
|
500
|
+
|
|
501
|
+
return ReferencedRaster(
|
|
502
|
+
rasterize(
|
|
503
|
+
[
|
|
504
|
+
reproject_geometry(
|
|
505
|
+
footprint,
|
|
506
|
+
self.crs,
|
|
507
|
+
grid.crs,
|
|
508
|
+
# CRS Bounds are sometimes smaller than (Mapchete) Grid Bounds,
|
|
509
|
+
# if clipping allowed it will mask out features at CRS Bounds border,
|
|
510
|
+
# therefore clip_to_crs_bounds: False; see mapchete.geometry.reproject reproject_geometry
|
|
511
|
+
clip_to_crs_bounds=False,
|
|
512
|
+
)
|
|
513
|
+
],
|
|
514
|
+
out_shape=grid.shape,
|
|
515
|
+
transform=grid.transform,
|
|
516
|
+
all_touched=True,
|
|
517
|
+
fill=1,
|
|
518
|
+
default_value=0,
|
|
519
|
+
).astype(bool),
|
|
520
|
+
transform=grid.transform,
|
|
521
|
+
bounds=grid.bounds,
|
|
522
|
+
crs=grid.crs,
|
|
523
|
+
)
|
|
524
|
+
|
|
525
|
+
def get_mask(
|
|
526
|
+
self,
|
|
527
|
+
grid: Union[GridProtocol, Resolution] = Resolution["10m"],
|
|
528
|
+
mask_config: MaskConfig = MaskConfig(),
|
|
529
|
+
target_mask: Optional[np.ndarray] = None,
|
|
530
|
+
) -> ReferencedRaster:
|
|
531
|
+
"""
|
|
532
|
+
Merge all configured masks into one.
|
|
533
|
+
"""
|
|
534
|
+
grid = (
|
|
535
|
+
self.metadata.grid(grid)
|
|
536
|
+
if isinstance(grid, Resolution)
|
|
537
|
+
else Grid.from_obj(grid)
|
|
538
|
+
)
|
|
539
|
+
if target_mask is None:
|
|
540
|
+
target_mask = np.zeros(shape=grid.shape, dtype=bool)
|
|
541
|
+
else:
|
|
542
|
+
if target_mask.shape != grid.shape:
|
|
543
|
+
raise ValueError("a target mask must have the same shape as the grid")
|
|
544
|
+
logger.debug("got custom target mask to start with: %s", target_mask.shape)
|
|
545
|
+
|
|
546
|
+
def _check_full(arr):
|
|
547
|
+
# ATTENTION: target_mask and out have to be combined *after* mask was buffered!
|
|
548
|
+
# use 'logical or' not '+' !!!
|
|
549
|
+
if (arr | target_mask).all():
|
|
550
|
+
raise AllMasked()
|
|
551
|
+
|
|
552
|
+
out = np.zeros(shape=grid.shape, dtype=bool)
|
|
553
|
+
logger.debug("generate mask for product %s ...", str(self))
|
|
554
|
+
try:
|
|
555
|
+
_check_full(out)
|
|
556
|
+
if mask_config.footprint:
|
|
557
|
+
logger.debug("generate footprint nodata mask ...")
|
|
558
|
+
try:
|
|
559
|
+
out |= self.footprint_nodata_mask(
|
|
560
|
+
grid, buffer_m=mask_config.footprint_buffer_m
|
|
561
|
+
).data
|
|
562
|
+
_check_full(out)
|
|
563
|
+
except EmptyFootprintException:
|
|
564
|
+
raise AllMasked()
|
|
565
|
+
if mask_config.l1c_cloud_type:
|
|
566
|
+
logger.debug("generate L1C mask ...")
|
|
567
|
+
out |= self.read_l1c_cloud_mask(
|
|
568
|
+
grid,
|
|
569
|
+
mask_config.l1c_cloud_type,
|
|
570
|
+
cached_read=mask_config.l1c_cloud_mask_cached_read,
|
|
571
|
+
).data
|
|
572
|
+
_check_full(out)
|
|
573
|
+
if mask_config.cloud_probability_threshold != 100:
|
|
574
|
+
logger.debug(
|
|
575
|
+
"generate cloud probability (%s) mask ...",
|
|
576
|
+
mask_config.cloud_probability_threshold,
|
|
577
|
+
)
|
|
578
|
+
cld_prb = self.read_cloud_probability(
|
|
579
|
+
grid,
|
|
580
|
+
from_resolution=mask_config.cloud_probability_resolution,
|
|
581
|
+
cached_read=mask_config.cloud_probability_cached_read,
|
|
582
|
+
).data
|
|
583
|
+
out |= np.where(
|
|
584
|
+
cld_prb >= mask_config.cloud_probability_threshold, True, False
|
|
585
|
+
)
|
|
586
|
+
_check_full(out)
|
|
587
|
+
if mask_config.scl_classes:
|
|
588
|
+
logger.debug(
|
|
589
|
+
"generate SCL mask using %s ...",
|
|
590
|
+
", ".join(
|
|
591
|
+
[scl_class.name for scl_class in mask_config.scl_classes]
|
|
592
|
+
),
|
|
593
|
+
)
|
|
594
|
+
# convert SCL classes to pixel values
|
|
595
|
+
scl_values = [scl.value for scl in mask_config.scl_classes]
|
|
596
|
+
# read SCL mask
|
|
597
|
+
scl_arr = self.read_scl(
|
|
598
|
+
grid, cached_read=mask_config.scl_cached_read
|
|
599
|
+
).data
|
|
600
|
+
# mask out specific pixel values
|
|
601
|
+
out |= np.isin(scl_arr, scl_values)
|
|
602
|
+
_check_full(out)
|
|
603
|
+
if mask_config.snow_ice:
|
|
604
|
+
logger.debug("generate snow & ice mask ...")
|
|
605
|
+
out |= self.read_snow_ice_mask(
|
|
606
|
+
grid, cached_read=mask_config.snow_ice_mask_cached_read
|
|
607
|
+
).data
|
|
608
|
+
_check_full(out)
|
|
609
|
+
if mask_config.snow_probability_threshold != 100:
|
|
610
|
+
logger.debug(
|
|
611
|
+
"generate snow probability (%s) mask ...",
|
|
612
|
+
mask_config.snow_probability_threshold,
|
|
613
|
+
)
|
|
614
|
+
snw_prb = self.read_snow_probability(
|
|
615
|
+
grid,
|
|
616
|
+
from_resolution=mask_config.snow_probability_resolution,
|
|
617
|
+
cached_read=mask_config.snow_probability_cached_read,
|
|
618
|
+
).data
|
|
619
|
+
out |= np.where(
|
|
620
|
+
snw_prb >= mask_config.snow_probability_threshold, True, False
|
|
621
|
+
)
|
|
622
|
+
_check_full(out)
|
|
623
|
+
if mask_config.buffer:
|
|
624
|
+
logger.debug(
|
|
625
|
+
"apply buffer (%s) to combined mask ...", mask_config.buffer
|
|
626
|
+
)
|
|
627
|
+
out = buffer_array(array=out, buffer=mask_config.buffer)
|
|
628
|
+
_check_full(out)
|
|
629
|
+
except AllMasked:
|
|
630
|
+
logger.debug(
|
|
631
|
+
"mask for product %s already full, skip reading other masks", self.id
|
|
632
|
+
)
|
|
633
|
+
except FileNotFoundError as exc: # pragma: no cover
|
|
634
|
+
raise CorruptedProduct from exc
|
|
635
|
+
|
|
636
|
+
# ATTENTION: target_mask and out have to be combined *after* mask was buffered!
|
|
637
|
+
# use 'logical or' not '+' !!!
|
|
638
|
+
return ReferencedRaster(
|
|
639
|
+
out | target_mask,
|
|
640
|
+
transform=grid.transform,
|
|
641
|
+
crs=grid.crs,
|
|
642
|
+
bounds=grid.bounds,
|
|
643
|
+
)
|
|
644
|
+
|
|
645
|
+
def get_property(self, name: str) -> Any:
|
|
646
|
+
if name not in self._item_property_cache:
|
|
647
|
+
self._item_property_cache[name] = get_item_property(self.item, name)
|
|
648
|
+
return self._item_property_cache[name]
|
|
649
|
+
|
|
650
|
+
def _apply_sentinel2_bandpass_adjustment(
|
|
651
|
+
self, uncorrected: ma.MaskedArray, assets: List[str], computing_dtype=np.float32
|
|
652
|
+
) -> ma.MaskedArray:
|
|
653
|
+
out_arr: ma.MaskedArray = ma.masked_array(
|
|
654
|
+
data=np.zeros(uncorrected.shape, uncorrected.dtype),
|
|
655
|
+
mask=uncorrected.mask.copy(),
|
|
656
|
+
fill_value=uncorrected.fill_value,
|
|
657
|
+
)
|
|
658
|
+
for band_idx, asset in enumerate(assets):
|
|
659
|
+
out_arr[band_idx] = apply_bandpass_adjustment(
|
|
660
|
+
uncorrected[band_idx],
|
|
661
|
+
item=self.item,
|
|
662
|
+
l2a_band=asset_name_to_l2a_band(self.item, asset),
|
|
663
|
+
computing_dtype=computing_dtype,
|
|
664
|
+
out_dtype=uncorrected.dtype,
|
|
665
|
+
)
|
|
666
|
+
return out_arr
|
|
667
|
+
|
|
668
|
+
def _apply_brdf(
|
|
669
|
+
self,
|
|
670
|
+
uncorrected: ma.MaskedArray,
|
|
671
|
+
assets: List[str],
|
|
672
|
+
brdf_config: BRDFConfig,
|
|
673
|
+
grid: Union[GridProtocol, Resolution, None] = Resolution["10m"],
|
|
674
|
+
resampling: Resampling = Resampling.nearest,
|
|
675
|
+
mask_config: MaskConfig = MaskConfig(),
|
|
676
|
+
) -> ma.MaskedArray:
|
|
677
|
+
out_arr: ma.MaskedArray = ma.masked_array(
|
|
678
|
+
data=np.zeros(uncorrected.shape, uncorrected.dtype),
|
|
679
|
+
mask=uncorrected.mask.copy(),
|
|
680
|
+
fill_value=uncorrected.fill_value,
|
|
681
|
+
)
|
|
682
|
+
|
|
683
|
+
# apply default correction defined in root
|
|
684
|
+
if brdf_config.model == BRDFModels.none:
|
|
685
|
+
logger.debug("no default BRDF model specified")
|
|
686
|
+
out_arr[:] = uncorrected
|
|
687
|
+
else:
|
|
688
|
+
logger.debug("applying %s to bands", brdf_config.model)
|
|
689
|
+
for band_idx, asset in enumerate(assets):
|
|
690
|
+
out_arr[band_idx] = apply_correction(
|
|
691
|
+
band=uncorrected[band_idx],
|
|
692
|
+
correction=self.read_brdf_grid(
|
|
693
|
+
asset_name_to_l2a_band(self.item, asset),
|
|
694
|
+
resampling=resampling,
|
|
695
|
+
grid=grid,
|
|
696
|
+
brdf_config=brdf_config,
|
|
697
|
+
),
|
|
698
|
+
correction_weight=brdf_config.correction_weight,
|
|
699
|
+
log10_bands_scale=brdf_config.log10_bands_scale,
|
|
700
|
+
)
|
|
701
|
+
|
|
702
|
+
# if SCL-specific correction is configured, apply and overwrite values in array
|
|
703
|
+
if brdf_config.scl_specific_configurations:
|
|
704
|
+
logger.debug("SCL class specific BRDF correction required")
|
|
705
|
+
scl_arr = self.read_scl(grid, mask_config.scl_cached_read).data
|
|
706
|
+
|
|
707
|
+
for scl_config in brdf_config.scl_specific_configurations:
|
|
708
|
+
scl_mask = np.isin(
|
|
709
|
+
scl_arr, [scl_class.value for scl_class in scl_config.scl_classes]
|
|
710
|
+
)
|
|
711
|
+
|
|
712
|
+
for band_idx, asset in enumerate(assets):
|
|
713
|
+
if scl_config.model == BRDFModels.none:
|
|
714
|
+
# use uncorrected values from original array
|
|
715
|
+
out_arr[band_idx][scl_mask] = uncorrected[band_idx][scl_mask]
|
|
716
|
+
|
|
717
|
+
elif scl_mask.any():
|
|
718
|
+
logger.debug(
|
|
719
|
+
"applying BRDF model %s to SCL classes %s",
|
|
720
|
+
scl_config.model.value,
|
|
721
|
+
", ".join(
|
|
722
|
+
[scl_class.name for scl_class in scl_config.scl_classes]
|
|
723
|
+
),
|
|
724
|
+
)
|
|
725
|
+
# apply correction band by band
|
|
726
|
+
out_arr[band_idx][scl_mask] = apply_correction(
|
|
727
|
+
uncorrected[band_idx],
|
|
728
|
+
self.read_brdf_grid(
|
|
729
|
+
asset_name_to_l2a_band(self.item, asset),
|
|
730
|
+
resampling=resampling,
|
|
731
|
+
grid=grid,
|
|
732
|
+
brdf_config=scl_config,
|
|
733
|
+
),
|
|
734
|
+
correction_weight=scl_config.correction_weight,
|
|
735
|
+
log10_bands_scale=scl_config.log10_bands_scale,
|
|
736
|
+
)[scl_mask]
|
|
737
|
+
|
|
738
|
+
# leave it be for all other cases
|
|
739
|
+
|
|
740
|
+
return out_arr
|
|
741
|
+
|
|
742
|
+
|
|
743
|
+
def asset_name_to_l2a_band(item: Item, asset_name: str) -> L2ABand:
|
|
744
|
+
asset = item.assets[asset_name]
|
|
745
|
+
asset_path = MPath(asset.href)
|
|
746
|
+
band_name = asset_path.name.split(".")[0]
|
|
747
|
+
return L2ABand[band_name]
|