mapchete-eo 2026.2.0__tar.gz → 2026.11.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {mapchete_eo-2026.2.0 → mapchete_eo-2026.11.0}/PKG-INFO +5 -3
- mapchete_eo-2026.11.0/mapchete_eo/__init__.py +1 -0
- {mapchete_eo-2026.2.0 → mapchete_eo-2026.11.0}/mapchete_eo/base.py +22 -6
- {mapchete_eo-2026.2.0 → mapchete_eo-2026.11.0}/mapchete_eo/io/assets.py +5 -5
- {mapchete_eo-2026.2.0 → mapchete_eo-2026.11.0}/mapchete_eo/io/levelled_cubes.py +9 -6
- {mapchete_eo-2026.2.0 → mapchete_eo-2026.11.0}/mapchete_eo/io/path.py +1 -21
- {mapchete_eo-2026.2.0 → mapchete_eo-2026.11.0}/mapchete_eo/io/products.py +20 -8
- {mapchete_eo-2026.2.0 → mapchete_eo-2026.11.0}/mapchete_eo/platforms/sentinel2/masks.py +6 -3
- {mapchete_eo-2026.2.0 → mapchete_eo-2026.11.0}/mapchete_eo/platforms/sentinel2/preconfigured_sources/item_mappers.py +2 -2
- {mapchete_eo-2026.2.0 → mapchete_eo-2026.11.0}/mapchete_eo/platforms/sentinel2/product.py +4 -2
- {mapchete_eo-2026.2.0 → mapchete_eo-2026.11.0}/mapchete_eo/search/stac_search.py +1 -1
- {mapchete_eo-2026.2.0 → mapchete_eo-2026.11.0}/pyproject.toml +18 -5
- mapchete_eo-2026.2.0/mapchete_eo/__init__.py +0 -1
- {mapchete_eo-2026.2.0 → mapchete_eo-2026.11.0}/.gitignore +0 -0
- {mapchete_eo-2026.2.0 → mapchete_eo-2026.11.0}/LICENSE +0 -0
- {mapchete_eo-2026.2.0 → mapchete_eo-2026.11.0}/README.rst +0 -0
- {mapchete_eo-2026.2.0 → mapchete_eo-2026.11.0}/mapchete_eo/array/__init__.py +0 -0
- {mapchete_eo-2026.2.0 → mapchete_eo-2026.11.0}/mapchete_eo/array/buffer.py +0 -0
- {mapchete_eo-2026.2.0 → mapchete_eo-2026.11.0}/mapchete_eo/array/color.py +0 -0
- {mapchete_eo-2026.2.0 → mapchete_eo-2026.11.0}/mapchete_eo/array/convert.py +0 -0
- {mapchete_eo-2026.2.0 → mapchete_eo-2026.11.0}/mapchete_eo/blacklist.txt +0 -0
- {mapchete_eo-2026.2.0 → mapchete_eo-2026.11.0}/mapchete_eo/cli/__init__.py +0 -0
- {mapchete_eo-2026.2.0 → mapchete_eo-2026.11.0}/mapchete_eo/cli/bounds.py +0 -0
- {mapchete_eo-2026.2.0 → mapchete_eo-2026.11.0}/mapchete_eo/cli/options_arguments.py +0 -0
- {mapchete_eo-2026.2.0 → mapchete_eo-2026.11.0}/mapchete_eo/cli/s2_brdf.py +0 -0
- {mapchete_eo-2026.2.0 → mapchete_eo-2026.11.0}/mapchete_eo/cli/s2_cat_results.py +0 -0
- {mapchete_eo-2026.2.0 → mapchete_eo-2026.11.0}/mapchete_eo/cli/s2_find_broken_products.py +0 -0
- {mapchete_eo-2026.2.0 → mapchete_eo-2026.11.0}/mapchete_eo/cli/s2_jp2_static_catalog.py +0 -0
- {mapchete_eo-2026.2.0 → mapchete_eo-2026.11.0}/mapchete_eo/cli/s2_mask.py +0 -0
- {mapchete_eo-2026.2.0 → mapchete_eo-2026.11.0}/mapchete_eo/cli/s2_mgrs.py +0 -0
- {mapchete_eo-2026.2.0 → mapchete_eo-2026.11.0}/mapchete_eo/cli/s2_rgb.py +0 -0
- {mapchete_eo-2026.2.0 → mapchete_eo-2026.11.0}/mapchete_eo/cli/s2_verify.py +0 -0
- {mapchete_eo-2026.2.0 → mapchete_eo-2026.11.0}/mapchete_eo/cli/static_catalog.py +0 -0
- {mapchete_eo-2026.2.0 → mapchete_eo-2026.11.0}/mapchete_eo/eostac.py +0 -0
- {mapchete_eo-2026.2.0 → mapchete_eo-2026.11.0}/mapchete_eo/exceptions.py +0 -0
- {mapchete_eo-2026.2.0 → mapchete_eo-2026.11.0}/mapchete_eo/image_operations/__init__.py +0 -0
- {mapchete_eo-2026.2.0 → mapchete_eo-2026.11.0}/mapchete_eo/image_operations/blend_functions.py +0 -0
- {mapchete_eo-2026.2.0 → mapchete_eo-2026.11.0}/mapchete_eo/image_operations/color_correction.py +0 -0
- {mapchete_eo-2026.2.0 → mapchete_eo-2026.11.0}/mapchete_eo/image_operations/compositing.py +0 -0
- {mapchete_eo-2026.2.0 → mapchete_eo-2026.11.0}/mapchete_eo/image_operations/dtype_scale.py +0 -0
- {mapchete_eo-2026.2.0 → mapchete_eo-2026.11.0}/mapchete_eo/image_operations/fillnodata.py +0 -0
- {mapchete_eo-2026.2.0 → mapchete_eo-2026.11.0}/mapchete_eo/image_operations/filters.py +0 -0
- {mapchete_eo-2026.2.0 → mapchete_eo-2026.11.0}/mapchete_eo/image_operations/linear_normalization.py +0 -0
- {mapchete_eo-2026.2.0 → mapchete_eo-2026.11.0}/mapchete_eo/image_operations/sigmoidal.py +0 -0
- {mapchete_eo-2026.2.0 → mapchete_eo-2026.11.0}/mapchete_eo/io/__init__.py +0 -0
- {mapchete_eo-2026.2.0 → mapchete_eo-2026.11.0}/mapchete_eo/io/items.py +0 -0
- {mapchete_eo-2026.2.0 → mapchete_eo-2026.11.0}/mapchete_eo/io/profiles.py +0 -0
- {mapchete_eo-2026.2.0 → mapchete_eo-2026.11.0}/mapchete_eo/platforms/sentinel2/__init__.py +0 -0
- {mapchete_eo-2026.2.0 → mapchete_eo-2026.11.0}/mapchete_eo/platforms/sentinel2/_mapper_registry.py +0 -0
- {mapchete_eo-2026.2.0 → mapchete_eo-2026.11.0}/mapchete_eo/platforms/sentinel2/bandpass_adjustment.py +0 -0
- {mapchete_eo-2026.2.0 → mapchete_eo-2026.11.0}/mapchete_eo/platforms/sentinel2/brdf/__init__.py +0 -0
- {mapchete_eo-2026.2.0 → mapchete_eo-2026.11.0}/mapchete_eo/platforms/sentinel2/brdf/config.py +0 -0
- {mapchete_eo-2026.2.0 → mapchete_eo-2026.11.0}/mapchete_eo/platforms/sentinel2/brdf/correction.py +0 -0
- {mapchete_eo-2026.2.0 → mapchete_eo-2026.11.0}/mapchete_eo/platforms/sentinel2/brdf/hls.py +0 -0
- {mapchete_eo-2026.2.0 → mapchete_eo-2026.11.0}/mapchete_eo/platforms/sentinel2/brdf/models.py +0 -0
- {mapchete_eo-2026.2.0 → mapchete_eo-2026.11.0}/mapchete_eo/platforms/sentinel2/brdf/protocols.py +0 -0
- {mapchete_eo-2026.2.0 → mapchete_eo-2026.11.0}/mapchete_eo/platforms/sentinel2/brdf/ross_thick.py +0 -0
- {mapchete_eo-2026.2.0 → mapchete_eo-2026.11.0}/mapchete_eo/platforms/sentinel2/brdf/sun_angle_arrays.py +0 -0
- {mapchete_eo-2026.2.0 → mapchete_eo-2026.11.0}/mapchete_eo/platforms/sentinel2/config.py +0 -0
- {mapchete_eo-2026.2.0 → mapchete_eo-2026.11.0}/mapchete_eo/platforms/sentinel2/driver.py +0 -0
- {mapchete_eo-2026.2.0 → mapchete_eo-2026.11.0}/mapchete_eo/platforms/sentinel2/metadata_parser/__init__.py +0 -0
- {mapchete_eo-2026.2.0 → mapchete_eo-2026.11.0}/mapchete_eo/platforms/sentinel2/metadata_parser/base.py +0 -0
- {mapchete_eo-2026.2.0 → mapchete_eo-2026.11.0}/mapchete_eo/platforms/sentinel2/metadata_parser/default_path_mapper.py +0 -0
- {mapchete_eo-2026.2.0 → mapchete_eo-2026.11.0}/mapchete_eo/platforms/sentinel2/metadata_parser/models.py +0 -0
- {mapchete_eo-2026.2.0 → mapchete_eo-2026.11.0}/mapchete_eo/platforms/sentinel2/metadata_parser/s2metadata.py +0 -0
- {mapchete_eo-2026.2.0 → mapchete_eo-2026.11.0}/mapchete_eo/platforms/sentinel2/preconfigured_sources/__init__.py +0 -0
- {mapchete_eo-2026.2.0 → mapchete_eo-2026.11.0}/mapchete_eo/platforms/sentinel2/preconfigured_sources/guessers.py +0 -0
- {mapchete_eo-2026.2.0 → mapchete_eo-2026.11.0}/mapchete_eo/platforms/sentinel2/preconfigured_sources/metadata_xml_mappers.py +0 -0
- {mapchete_eo-2026.2.0 → mapchete_eo-2026.11.0}/mapchete_eo/platforms/sentinel2/preprocessing_tasks.py +0 -0
- {mapchete_eo-2026.2.0 → mapchete_eo-2026.11.0}/mapchete_eo/platforms/sentinel2/processing_baseline.py +0 -0
- {mapchete_eo-2026.2.0 → mapchete_eo-2026.11.0}/mapchete_eo/platforms/sentinel2/source.py +0 -0
- {mapchete_eo-2026.2.0 → mapchete_eo-2026.11.0}/mapchete_eo/platforms/sentinel2/types.py +0 -0
- {mapchete_eo-2026.2.0 → mapchete_eo-2026.11.0}/mapchete_eo/processes/__init__.py +0 -0
- {mapchete_eo-2026.2.0 → mapchete_eo-2026.11.0}/mapchete_eo/processes/config.py +0 -0
- {mapchete_eo-2026.2.0 → mapchete_eo-2026.11.0}/mapchete_eo/processes/dtype_scale.py +0 -0
- {mapchete_eo-2026.2.0 → mapchete_eo-2026.11.0}/mapchete_eo/processes/eo_to_xarray.py +0 -0
- {mapchete_eo-2026.2.0 → mapchete_eo-2026.11.0}/mapchete_eo/processes/merge_rasters.py +0 -0
- {mapchete_eo-2026.2.0 → mapchete_eo-2026.11.0}/mapchete_eo/product.py +0 -0
- {mapchete_eo-2026.2.0 → mapchete_eo-2026.11.0}/mapchete_eo/protocols.py +0 -0
- {mapchete_eo-2026.2.0 → mapchete_eo-2026.11.0}/mapchete_eo/search/__init__.py +0 -0
- {mapchete_eo-2026.2.0 → mapchete_eo-2026.11.0}/mapchete_eo/search/base.py +0 -0
- {mapchete_eo-2026.2.0 → mapchete_eo-2026.11.0}/mapchete_eo/search/config.py +0 -0
- {mapchete_eo-2026.2.0 → mapchete_eo-2026.11.0}/mapchete_eo/search/s2_mgrs.py +0 -0
- {mapchete_eo-2026.2.0 → mapchete_eo-2026.11.0}/mapchete_eo/search/stac_static.py +0 -0
- {mapchete_eo-2026.2.0 → mapchete_eo-2026.11.0}/mapchete_eo/search/utm_search.py +0 -0
- {mapchete_eo-2026.2.0 → mapchete_eo-2026.11.0}/mapchete_eo/settings.py +0 -0
- {mapchete_eo-2026.2.0 → mapchete_eo-2026.11.0}/mapchete_eo/sort.py +0 -0
- {mapchete_eo-2026.2.0 → mapchete_eo-2026.11.0}/mapchete_eo/source.py +0 -0
- {mapchete_eo-2026.2.0 → mapchete_eo-2026.11.0}/mapchete_eo/time.py +0 -0
- {mapchete_eo-2026.2.0 → mapchete_eo-2026.11.0}/mapchete_eo/types.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
|
-
Metadata-Version: 2.
|
|
1
|
+
Metadata-Version: 2.5
|
|
2
2
|
Name: mapchete-eo
|
|
3
|
-
Version: 2026.
|
|
3
|
+
Version: 2026.11.0
|
|
4
4
|
Summary: mapchete EO data reader
|
|
5
5
|
Project-URL: Homepage, https://gitlab.eox.at/maps/mapchete_eo
|
|
6
6
|
Author-email: Joachim Ungar <joachim.ungar@eox.at>, Petr Sevcik <petr.sevcik@eox.at>
|
|
@@ -35,9 +35,11 @@ Requires-Dist: myst-parser; extra == 'docs'
|
|
|
35
35
|
Requires-Dist: sphinx-rtd-theme>=3.0.2; extra == 'docs'
|
|
36
36
|
Requires-Dist: sphinx>=8.0; extra == 'docs'
|
|
37
37
|
Provides-Extra: test
|
|
38
|
+
Requires-Dist: httpx; extra == 'test'
|
|
38
39
|
Requires-Dist: pytest; extra == 'test'
|
|
39
|
-
Requires-Dist: pytest-
|
|
40
|
+
Requires-Dist: pytest-cov; extra == 'test'
|
|
40
41
|
Requires-Dist: pytest-lazy-fixtures; extra == 'test'
|
|
42
|
+
Requires-Dist: pytest-split; extra == 'test'
|
|
41
43
|
Description-Content-Type: text/x-rst
|
|
42
44
|
|
|
43
45
|
.. image:: logo/mapchete_eo.svg
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
__version__ = "2026.11.0"
|
|
@@ -162,6 +162,7 @@ class EODataCube(base.InputTile):
|
|
|
162
162
|
sort: Optional[SortMethodConfig] = None,
|
|
163
163
|
nodatavals: NodataVals = None,
|
|
164
164
|
raise_empty: bool = True,
|
|
165
|
+
clear_caches: bool = True,
|
|
165
166
|
**kwargs,
|
|
166
167
|
) -> xr.Dataset:
|
|
167
168
|
"""
|
|
@@ -180,6 +181,7 @@ class EODataCube(base.InputTile):
|
|
|
180
181
|
raise_empty=raise_empty,
|
|
181
182
|
product_read_kwargs=kwargs,
|
|
182
183
|
sort=sort,
|
|
184
|
+
clear_caches=clear_caches,
|
|
183
185
|
**self.default_read_values(
|
|
184
186
|
merge_products_by=merge_products_by,
|
|
185
187
|
merge_method=merge_method,
|
|
@@ -202,6 +204,7 @@ class EODataCube(base.InputTile):
|
|
|
202
204
|
sort: Optional[SortMethodConfig] = None,
|
|
203
205
|
nodatavals: NodataVals = None,
|
|
204
206
|
raise_empty: bool = True,
|
|
207
|
+
clear_caches: bool = True,
|
|
205
208
|
**kwargs,
|
|
206
209
|
) -> ma.MaskedArray:
|
|
207
210
|
"""
|
|
@@ -220,6 +223,7 @@ class EODataCube(base.InputTile):
|
|
|
220
223
|
product_read_kwargs=kwargs,
|
|
221
224
|
raise_empty=raise_empty,
|
|
222
225
|
sort=sort,
|
|
226
|
+
clear_caches=clear_caches,
|
|
223
227
|
**self.default_read_values(
|
|
224
228
|
merge_products_by=merge_products_by,
|
|
225
229
|
merge_method=merge_method,
|
|
@@ -247,6 +251,7 @@ class EODataCube(base.InputTile):
|
|
|
247
251
|
band_axis_name: str = "bands",
|
|
248
252
|
x_axis_name: str = "x",
|
|
249
253
|
y_axis_name: str = "y",
|
|
254
|
+
clear_caches: bool = True,
|
|
250
255
|
**kwargs,
|
|
251
256
|
) -> xr.Dataset:
|
|
252
257
|
return read_levelled_cube_to_xarray(
|
|
@@ -267,6 +272,7 @@ class EODataCube(base.InputTile):
|
|
|
267
272
|
x_axis_name=x_axis_name,
|
|
268
273
|
y_axis_name=y_axis_name,
|
|
269
274
|
sort=sort,
|
|
275
|
+
clear_caches=clear_caches,
|
|
270
276
|
**self.default_read_values(
|
|
271
277
|
merge_products_by=merge_products_by,
|
|
272
278
|
merge_method=merge_method,
|
|
@@ -290,6 +296,7 @@ class EODataCube(base.InputTile):
|
|
|
290
296
|
merge_method: Optional[MergeMethod] = None,
|
|
291
297
|
sort: SortMethodConfig = TargetDateSort(),
|
|
292
298
|
raise_empty: bool = True,
|
|
299
|
+
clear_caches: bool = True,
|
|
293
300
|
**kwargs,
|
|
294
301
|
) -> ma.MaskedArray:
|
|
295
302
|
"""
|
|
@@ -327,6 +334,7 @@ class EODataCube(base.InputTile):
|
|
|
327
334
|
raise_empty=raise_empty,
|
|
328
335
|
product_read_kwargs=kwargs,
|
|
329
336
|
sort=sort,
|
|
337
|
+
clear_caches=clear_caches,
|
|
330
338
|
**self.default_read_values(
|
|
331
339
|
merge_products_by=merge_products_by,
|
|
332
340
|
merge_method=merge_method,
|
|
@@ -500,12 +508,6 @@ class InputData(base.InputData):
|
|
|
500
508
|
self.area = self._init_area(input_params)
|
|
501
509
|
self.time = self.params.time
|
|
502
510
|
|
|
503
|
-
self.eo_bands = [
|
|
504
|
-
eo_band
|
|
505
|
-
for source in self.params.source
|
|
506
|
-
for eo_band in source.eo_bands(base_dir=self.conf_dir)
|
|
507
|
-
]
|
|
508
|
-
|
|
509
511
|
if self.readonly: # pragma: no cover
|
|
510
512
|
return
|
|
511
513
|
# don't use preprocessing tasks for Sentinel-2 products:
|
|
@@ -533,6 +535,17 @@ class InputData(base.InputData):
|
|
|
533
535
|
]
|
|
534
536
|
)
|
|
535
537
|
|
|
538
|
+
@cached_property
|
|
539
|
+
def eo_bands(self) -> List[str]:
|
|
540
|
+
if self.area.is_empty:
|
|
541
|
+
logger.debug("Input area is empty, nothing to be found here.")
|
|
542
|
+
return []
|
|
543
|
+
return [
|
|
544
|
+
eo_band
|
|
545
|
+
for source in self.params.source
|
|
546
|
+
for eo_band in source.eo_bands(base_dir=self.conf_dir)
|
|
547
|
+
]
|
|
548
|
+
|
|
536
549
|
def _init_area(self, input_params: dict) -> BaseGeometry:
|
|
537
550
|
"""Returns valid driver area for this process."""
|
|
538
551
|
process_area = input_params["delimiters"]["effective_area"]
|
|
@@ -556,6 +569,9 @@ class InputData(base.InputData):
|
|
|
556
569
|
return process_area
|
|
557
570
|
|
|
558
571
|
def source_items(self) -> Generator[Item, None, None]:
|
|
572
|
+
if self.area.is_empty:
|
|
573
|
+
logger.debug("Input area is empty, nothing to be found here.")
|
|
574
|
+
return
|
|
559
575
|
already_returned = set()
|
|
560
576
|
for source in self.params.source:
|
|
561
577
|
area = reproject_geometry(
|
|
@@ -25,7 +25,7 @@ from rasterio.profiles import Profile
|
|
|
25
25
|
from rasterio.vrt import WarpedVRT
|
|
26
26
|
from retry import retry
|
|
27
27
|
|
|
28
|
-
from mapchete_eo.io.path import COMMON_RASTER_EXTENSIONS, asset_mpath
|
|
28
|
+
from mapchete_eo.io.path import COMMON_RASTER_EXTENSIONS, asset_mpath
|
|
29
29
|
from mapchete_eo.io.profiles import COGDeflateProfile
|
|
30
30
|
|
|
31
31
|
logger = logging.getLogger(__name__)
|
|
@@ -43,7 +43,7 @@ class STACRasterBandProperties(BaseModel):
|
|
|
43
43
|
nodataval: Optional[NodataVal] = None,
|
|
44
44
|
) -> STACRasterBandProperties:
|
|
45
45
|
if asset.extra_fields.get("raster:offset", {}):
|
|
46
|
-
properties = dict(
|
|
46
|
+
properties = dict( # pragma: no cover
|
|
47
47
|
offset=asset.extra_fields.get("raster:offset"),
|
|
48
48
|
scale=asset.extra_fields.get("raster:scale"),
|
|
49
49
|
nodata=asset.extra_fields.get("nodata", nodataval),
|
|
@@ -394,9 +394,9 @@ def should_be_converted(
|
|
|
394
394
|
return False
|
|
395
395
|
|
|
396
396
|
|
|
397
|
-
def _read_vector_mask(mask_path):
|
|
397
|
+
def _read_vector_mask(mask_path: MPath):
|
|
398
398
|
logger.debug("open %s with Fiona", mask_path)
|
|
399
|
-
with
|
|
399
|
+
with mask_path.local_copy() as cached:
|
|
400
400
|
try:
|
|
401
401
|
with fiona_open(cached) as src:
|
|
402
402
|
return list([dict(f) for f in src])
|
|
@@ -427,7 +427,7 @@ def read_mask_as_raster(
|
|
|
427
427
|
if dst_grid:
|
|
428
428
|
dst_grid = Grid.from_obj(dst_grid)
|
|
429
429
|
try:
|
|
430
|
-
with
|
|
430
|
+
with path.local_copy(active=cached_read) as path:
|
|
431
431
|
if path.suffix in COMMON_RASTER_EXTENSIONS:
|
|
432
432
|
if dst_grid:
|
|
433
433
|
array = read_raster_window(
|
|
@@ -41,6 +41,7 @@ def read_levelled_cube_to_np_array(
|
|
|
41
41
|
out_dtype: DTypeLike = np.uint16,
|
|
42
42
|
out_fill_value: NodataVal = 0,
|
|
43
43
|
read_mask: Optional[np.ndarray] = None,
|
|
44
|
+
clear_caches: bool = True,
|
|
44
45
|
) -> ma.MaskedArray:
|
|
45
46
|
"""
|
|
46
47
|
Read products as slices into a cube by filling up nodata gaps with next slice.
|
|
@@ -99,7 +100,7 @@ def read_levelled_cube_to_np_array(
|
|
|
99
100
|
slices_read_count, slices_skip_count = 0, 0
|
|
100
101
|
|
|
101
102
|
# pick slices one by one
|
|
102
|
-
for slice_count,
|
|
103
|
+
for slice_count, slice in enumerate(slices, 1):
|
|
103
104
|
# all filled up? let's get outta here!
|
|
104
105
|
if not out.mask.any():
|
|
105
106
|
logger.debug("cube has no pixels to be filled, quitting!")
|
|
@@ -113,11 +114,11 @@ def read_levelled_cube_to_np_array(
|
|
|
113
114
|
logger.debug(
|
|
114
115
|
"see if slice %s %s has some of the %s unmasked pixels for cube",
|
|
115
116
|
slice_count,
|
|
116
|
-
|
|
117
|
+
slice,
|
|
117
118
|
cube_nodata_mask.sum(),
|
|
118
119
|
)
|
|
119
|
-
with
|
|
120
|
-
slice_array =
|
|
120
|
+
with slice.cached(active=clear_caches):
|
|
121
|
+
slice_array = slice.read(
|
|
121
122
|
merge_method=merge_method,
|
|
122
123
|
product_read_kwargs=dict(
|
|
123
124
|
product_read_kwargs,
|
|
@@ -133,12 +134,12 @@ def read_levelled_cube_to_np_array(
|
|
|
133
134
|
)
|
|
134
135
|
slices_read_count += 1
|
|
135
136
|
except (EmptySliceException, CorruptedSlice) as exc:
|
|
136
|
-
logger.
|
|
137
|
+
logger.warning("skipped slice %s: %s", slice, str(exc))
|
|
137
138
|
slices_skip_count += 1
|
|
138
139
|
continue
|
|
139
140
|
|
|
140
141
|
# if slice was not empty, fill pixels into cube
|
|
141
|
-
logger.debug("add slice %s array to cube",
|
|
142
|
+
logger.debug("add slice %s array to cube", slice)
|
|
142
143
|
|
|
143
144
|
# iterate through layers of cube
|
|
144
145
|
for layer_index in range(target_height):
|
|
@@ -223,6 +224,7 @@ def read_levelled_cube_to_xarray(
|
|
|
223
224
|
x_axis_name: str = "x",
|
|
224
225
|
y_axis_name: str = "y",
|
|
225
226
|
read_mask: Optional[np.ndarray] = None,
|
|
227
|
+
clear_caches: bool = True,
|
|
226
228
|
) -> xr.Dataset:
|
|
227
229
|
"""
|
|
228
230
|
Read products as slices into a cube by filling up nodata gaps with next slice.
|
|
@@ -245,6 +247,7 @@ def read_levelled_cube_to_xarray(
|
|
|
245
247
|
product_read_kwargs=product_read_kwargs,
|
|
246
248
|
raise_empty=raise_empty,
|
|
247
249
|
read_mask=read_mask,
|
|
250
|
+
clear_caches=clear_caches,
|
|
248
251
|
),
|
|
249
252
|
slice_names=[f"layer-{ii}" for ii in range(target_height)],
|
|
250
253
|
band_names=variables,
|
|
@@ -1,14 +1,11 @@
|
|
|
1
1
|
import hashlib
|
|
2
2
|
import logging
|
|
3
|
-
from contextlib import contextmanager
|
|
4
3
|
from enum import Enum
|
|
5
|
-
from
|
|
6
|
-
from typing import Generator, Tuple, Union
|
|
4
|
+
from typing import Tuple, Union
|
|
7
5
|
from xml.etree.ElementTree import Element, fromstring
|
|
8
6
|
|
|
9
7
|
import fsspec
|
|
10
8
|
import pystac
|
|
11
|
-
from mapchete.io import copy
|
|
12
9
|
from mapchete.path import MPath
|
|
13
10
|
from mapchete.settings import IORetrySettings
|
|
14
11
|
from retry import retry
|
|
@@ -107,23 +104,6 @@ def path_in_paths(path, existing_paths) -> bool:
|
|
|
107
104
|
return False
|
|
108
105
|
|
|
109
106
|
|
|
110
|
-
@contextmanager
|
|
111
|
-
@retry(logger=logger, **dict(IORetrySettings()))
|
|
112
|
-
def cached_path(path: MPath, active: bool = True) -> Generator[MPath, None, None]:
|
|
113
|
-
"""If path is remote, download to temporary directory and return path."""
|
|
114
|
-
if active and path.is_remote():
|
|
115
|
-
with TemporaryDirectory() as tempdir:
|
|
116
|
-
tempfile = MPath(tempdir) / path.name
|
|
117
|
-
logger.debug("%s is remote, download to %s", path, tempfile)
|
|
118
|
-
copy(
|
|
119
|
-
path,
|
|
120
|
-
tempfile,
|
|
121
|
-
)
|
|
122
|
-
yield tempfile
|
|
123
|
-
else:
|
|
124
|
-
yield path
|
|
125
|
-
|
|
126
|
-
|
|
127
107
|
def asset_mpath(
|
|
128
108
|
item: pystac.Item,
|
|
129
109
|
asset: Union[str, Tuple[str, ...]],
|
|
@@ -51,6 +51,7 @@ def products_to_np_array(
|
|
|
51
51
|
raise_empty: bool = True,
|
|
52
52
|
out_dtype: Optional[DTypeLike] = None,
|
|
53
53
|
read_mask: Optional[np.ndarray] = None,
|
|
54
|
+
clear_caches: bool = True,
|
|
54
55
|
) -> ma.MaskedArray:
|
|
55
56
|
"""Read grid window of EOProducts and merge into a 4D xarray."""
|
|
56
57
|
return ma.stack(
|
|
@@ -69,6 +70,7 @@ def products_to_np_array(
|
|
|
69
70
|
product_read_kwargs=product_read_kwargs,
|
|
70
71
|
raise_empty=raise_empty,
|
|
71
72
|
read_mask=read_mask,
|
|
73
|
+
clear_caches=clear_caches,
|
|
72
74
|
)
|
|
73
75
|
]
|
|
74
76
|
)
|
|
@@ -91,6 +93,7 @@ def products_to_xarray(
|
|
|
91
93
|
raise_empty: bool = True,
|
|
92
94
|
product_read_kwargs: dict = {},
|
|
93
95
|
read_mask: Optional[np.ndarray] = None,
|
|
96
|
+
clear_caches: bool = True,
|
|
94
97
|
) -> xr.Dataset:
|
|
95
98
|
"""Read grid window of EOProducts and merge into a 4D xarray."""
|
|
96
99
|
data_vars = [
|
|
@@ -108,6 +111,7 @@ def products_to_xarray(
|
|
|
108
111
|
product_read_kwargs=product_read_kwargs,
|
|
109
112
|
raise_empty=raise_empty,
|
|
110
113
|
read_mask=read_mask,
|
|
114
|
+
clear_caches=clear_caches,
|
|
111
115
|
)
|
|
112
116
|
]
|
|
113
117
|
if merge_products_by and merge_products_by not in ["date", "datetime"]:
|
|
@@ -181,13 +185,20 @@ class Slice:
|
|
|
181
185
|
return properties
|
|
182
186
|
|
|
183
187
|
@contextmanager
|
|
184
|
-
def cached(self) -> Generator[Slice, None, None]:
|
|
188
|
+
def cached(self, active: bool = True) -> Generator[Slice, None, None]:
|
|
185
189
|
"""Clear caches and run garbage collector when context manager is closed."""
|
|
186
190
|
yield self
|
|
187
|
-
|
|
188
|
-
|
|
189
|
-
|
|
190
|
-
|
|
191
|
+
if active:
|
|
192
|
+
with Timer() as duration:
|
|
193
|
+
self.clear_cached_data()
|
|
194
|
+
# triggering the garbage collector helps fixing a memory leak on longer running
|
|
195
|
+
# process workers
|
|
196
|
+
cleared_objects = gc.collect()
|
|
197
|
+
logger.debug(
|
|
198
|
+
"Slice cache cleared and garbage collector cleared %s objects in %s",
|
|
199
|
+
cleared_objects,
|
|
200
|
+
duration,
|
|
201
|
+
)
|
|
191
202
|
|
|
192
203
|
def clear_cached_data(self):
|
|
193
204
|
logger.debug("clear caches of all products in slice")
|
|
@@ -360,6 +371,7 @@ def generate_slice_dataarrays(
|
|
|
360
371
|
product_read_kwargs: dict = {},
|
|
361
372
|
raise_empty: bool = True,
|
|
362
373
|
read_mask: Optional[np.ndarray] = None,
|
|
374
|
+
clear_caches: bool = True,
|
|
363
375
|
) -> Iterator[xr.DataArray]:
|
|
364
376
|
"""
|
|
365
377
|
Yield products or merged products into slices as DataArrays.
|
|
@@ -392,7 +404,7 @@ def generate_slice_dataarrays(
|
|
|
392
404
|
try:
|
|
393
405
|
# if merge_products_by is none, each slice contains just one product
|
|
394
406
|
# so nothing will have to be merged anyways
|
|
395
|
-
with slice.cached():
|
|
407
|
+
with slice.cached(active=clear_caches):
|
|
396
408
|
yield to_dataarray(
|
|
397
409
|
merge_products(
|
|
398
410
|
products=slice.products,
|
|
@@ -416,8 +428,8 @@ def generate_slice_dataarrays(
|
|
|
416
428
|
)
|
|
417
429
|
# if at least one slice can be yielded, the stack is not empty
|
|
418
430
|
stack_empty = False
|
|
419
|
-
except (EmptySliceException, CorruptedSlice) as
|
|
420
|
-
logger.warning(
|
|
431
|
+
except (EmptySliceException, CorruptedSlice) as exc:
|
|
432
|
+
logger.warning("skipped slice %s: %s", slice, str(exc))
|
|
421
433
|
|
|
422
434
|
if stack_empty:
|
|
423
435
|
raise EmptyStackException("all slices are empty")
|
|
@@ -60,6 +60,7 @@ def masks_to_xarray(
|
|
|
60
60
|
sort: Optional[SortMethodConfig] = None,
|
|
61
61
|
raise_empty: bool = True,
|
|
62
62
|
product_read_kwargs: dict = {},
|
|
63
|
+
clear_caches: bool = True,
|
|
63
64
|
) -> xr.Dataset:
|
|
64
65
|
"""
|
|
65
66
|
Read masks of products and merge into an xarray.Dataset.
|
|
@@ -74,6 +75,7 @@ def masks_to_xarray(
|
|
|
74
75
|
sort=sort,
|
|
75
76
|
product_read_kwargs=product_read_kwargs,
|
|
76
77
|
raise_empty=raise_empty,
|
|
78
|
+
clear_caches=clear_caches,
|
|
77
79
|
)
|
|
78
80
|
]
|
|
79
81
|
if merge_products_by and merge_products_by not in ["date", "datetime"]:
|
|
@@ -256,6 +258,7 @@ def generate_slice_masks_dataarrays(
|
|
|
256
258
|
mask_name: List = ["EOxCloudless_masks"],
|
|
257
259
|
product_read_kwargs: dict = {},
|
|
258
260
|
raise_empty: bool = True,
|
|
261
|
+
clear_caches: bool = True,
|
|
259
262
|
) -> Iterator[xr.DataArray]:
|
|
260
263
|
"""
|
|
261
264
|
Yield products or merged products into slices as DataArrays.
|
|
@@ -279,7 +282,7 @@ def generate_slice_masks_dataarrays(
|
|
|
279
282
|
try:
|
|
280
283
|
# if merge_products_by is none, each slice contains just one product
|
|
281
284
|
# so nothing will have to be merged anyways
|
|
282
|
-
with slice.cached():
|
|
285
|
+
with slice.cached(active=clear_caches):
|
|
283
286
|
yield to_dataarray(
|
|
284
287
|
merge_products_masks(
|
|
285
288
|
products=slice.products,
|
|
@@ -297,8 +300,8 @@ def generate_slice_masks_dataarrays(
|
|
|
297
300
|
)
|
|
298
301
|
# if at least one slice can be yielded, the stack is not empty
|
|
299
302
|
stack_empty = False
|
|
300
|
-
except (EmptySliceException, CorruptedSlice):
|
|
301
|
-
|
|
303
|
+
except (EmptySliceException, CorruptedSlice) as exc:
|
|
304
|
+
logger.warning("skipped slice %s: %s", slice, str(exc))
|
|
302
305
|
|
|
303
306
|
if stack_empty:
|
|
304
307
|
raise EmptyStackException("all slices are empty")
|
|
@@ -140,7 +140,7 @@ def map_cdse_paths_to_jp2_archive(item: Item) -> Item:
|
|
|
140
140
|
elif asset_name == "granule_metadata":
|
|
141
141
|
asset.href = str(product_basepath / "metadata.xml")
|
|
142
142
|
# change band asset names and point to their new locations
|
|
143
|
-
elif asset_name in CDSE_ASSET_NAME_MAPPING:
|
|
143
|
+
elif asset_name in CDSE_ASSET_NAME_MAPPING: # pragma: no cover
|
|
144
144
|
name, resolution = asset_name.split("_")
|
|
145
145
|
asset.href = product_basepath / f"R{resolution}" / f"{name}.jp2"
|
|
146
146
|
asset_name = CDSE_ASSET_NAME_MAPPING[asset_name]
|
|
@@ -155,7 +155,7 @@ def map_cdse_paths_to_jp2_archive(item: Item) -> Item:
|
|
|
155
155
|
|
|
156
156
|
@creates_s2metadata(from_collections=["CDSE"], to_metadata_archives=["CDSE"])
|
|
157
157
|
def cdse_s2metadata(item: Item) -> S2Metadata:
|
|
158
|
-
return S2Metadata.from_stac_item(
|
|
158
|
+
return S2Metadata.from_stac_item( # pragma: no cover
|
|
159
159
|
item,
|
|
160
160
|
path_mapper=CDSEPathMapper(MPath(item.assets["granule_metadata"].href)),
|
|
161
161
|
processing_baseline_field="processing:version",
|
|
@@ -201,7 +201,8 @@ class S2Product(EOProduct, EOProductProtocol):
|
|
|
201
201
|
|
|
202
202
|
@property
|
|
203
203
|
def item(self) -> Item:
|
|
204
|
-
if
|
|
204
|
+
if self._item is None:
|
|
205
|
+
logger.debug("%s no cached STAC Item found, recreating ...", self.id)
|
|
205
206
|
if self._item_dict:
|
|
206
207
|
self._item = Item.from_dict(self._item_dict)
|
|
207
208
|
else:
|
|
@@ -213,7 +214,8 @@ class S2Product(EOProduct, EOProductProtocol):
|
|
|
213
214
|
|
|
214
215
|
@property
|
|
215
216
|
def metadata(self) -> S2Metadata:
|
|
216
|
-
if
|
|
217
|
+
if self._metadata is None:
|
|
218
|
+
logger.debug("%s no cached metadata found, recreating ...", self.id)
|
|
217
219
|
if self._metadata_mapper:
|
|
218
220
|
self._metadata = self._metadata_mapper(self.item)
|
|
219
221
|
else:
|
|
@@ -36,7 +36,7 @@ class STACSearchCollection(StaticCollectionWriterMixin, CollectionSearcher):
|
|
|
36
36
|
def eo_bands(self) -> List[str]:
|
|
37
37
|
item_assets = self.client.extra_fields.get("item_assets", {})
|
|
38
38
|
for v in item_assets.values():
|
|
39
|
-
if "eo:bands" in v and "data" in v.get("roles", []):
|
|
39
|
+
if "eo:bands" in v and "data" in v.get("roles", []): # pragma: no cover
|
|
40
40
|
return ["eo:bands"]
|
|
41
41
|
else: # pragma: no cover
|
|
42
42
|
logger.debug("cannot find eo:bands definition from collections")
|
|
@@ -48,9 +48,11 @@ docs = [
|
|
|
48
48
|
"myst-parser",
|
|
49
49
|
]
|
|
50
50
|
test = [
|
|
51
|
+
"httpx",
|
|
51
52
|
"pytest",
|
|
52
|
-
"pytest-
|
|
53
|
-
"pytest-lazy-fixtures"
|
|
53
|
+
"pytest-cov",
|
|
54
|
+
"pytest-lazy-fixtures",
|
|
55
|
+
"pytest-split"
|
|
54
56
|
]
|
|
55
57
|
|
|
56
58
|
[project.entry-points."mapchete.cli.commands"]
|
|
@@ -81,10 +83,20 @@ ignore_missing_imports = true
|
|
|
81
83
|
|
|
82
84
|
[tool.coverage.run]
|
|
83
85
|
branch = true
|
|
86
|
+
relative_files = true
|
|
84
87
|
source = ["mapchete_eo"]
|
|
85
88
|
|
|
86
89
|
[tool.pytest.ini_options]
|
|
87
|
-
addopts =
|
|
90
|
+
addopts = [
|
|
91
|
+
"--durations=20",
|
|
92
|
+
"--verbose",
|
|
93
|
+
"--splitting-algorithm", "least_duration",
|
|
94
|
+
"--durations-path", "tests/.test_duration",
|
|
95
|
+
"--nf",
|
|
96
|
+
"--cov=mapchete_eo",
|
|
97
|
+
"--cov-report=term-missing:skip-covered",
|
|
98
|
+
"--cov-append"
|
|
99
|
+
]
|
|
88
100
|
testpaths = ["tests"]
|
|
89
101
|
markers = [
|
|
90
102
|
"remote: marks tests which require acces to remote resources (deselect with '-m \"not remote\"')",
|
|
@@ -93,5 +105,6 @@ markers = [
|
|
|
93
105
|
|
|
94
106
|
[tool.pyproject2conda]
|
|
95
107
|
channels = ["conda-forge"]
|
|
96
|
-
|
|
97
|
-
|
|
108
|
+
|
|
109
|
+
[tool.pyproject2conda.dependencies]
|
|
110
|
+
opencv-python-headless = { skip = true, packages = "opencv" }
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
__version__ = "2026.2.0"
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
{mapchete_eo-2026.2.0 → mapchete_eo-2026.11.0}/mapchete_eo/image_operations/blend_functions.py
RENAMED
|
File without changes
|
{mapchete_eo-2026.2.0 → mapchete_eo-2026.11.0}/mapchete_eo/image_operations/color_correction.py
RENAMED
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
{mapchete_eo-2026.2.0 → mapchete_eo-2026.11.0}/mapchete_eo/image_operations/linear_normalization.py
RENAMED
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
{mapchete_eo-2026.2.0 → mapchete_eo-2026.11.0}/mapchete_eo/platforms/sentinel2/_mapper_registry.py
RENAMED
|
File without changes
|
|
File without changes
|
{mapchete_eo-2026.2.0 → mapchete_eo-2026.11.0}/mapchete_eo/platforms/sentinel2/brdf/__init__.py
RENAMED
|
File without changes
|
{mapchete_eo-2026.2.0 → mapchete_eo-2026.11.0}/mapchete_eo/platforms/sentinel2/brdf/config.py
RENAMED
|
File without changes
|
{mapchete_eo-2026.2.0 → mapchete_eo-2026.11.0}/mapchete_eo/platforms/sentinel2/brdf/correction.py
RENAMED
|
File without changes
|
|
File without changes
|
{mapchete_eo-2026.2.0 → mapchete_eo-2026.11.0}/mapchete_eo/platforms/sentinel2/brdf/models.py
RENAMED
|
File without changes
|
{mapchete_eo-2026.2.0 → mapchete_eo-2026.11.0}/mapchete_eo/platforms/sentinel2/brdf/protocols.py
RENAMED
|
File without changes
|
{mapchete_eo-2026.2.0 → mapchete_eo-2026.11.0}/mapchete_eo/platforms/sentinel2/brdf/ross_thick.py
RENAMED
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|