mapchete-eo 2025.10.0__py2.py3-none-any.whl → 2025.11.0__py2.py3-none-any.whl

mapchete_eo/io/assets.py

@@ -82,7 +82,7 @@ def asset_to_np_array(
     path = asset_mpath(item, asset)
 
     # find out asset details if raster:bands is available
-    stac_raster_bands = STACRasterBandProperties.from_asset(
+    band_properties = STACRasterBandProperties.from_asset(
         item.assets[asset], nodataval=nodataval
     )
 
@@ -92,23 +92,23 @@ def asset_to_np_array(
         indexes=indexes,
         grid=grid,
         resampling=resampling.name,
-        dst_nodata=stac_raster_bands.nodata,
+        dst_nodata=band_properties.nodata,
     ).data
 
-    if apply_offset and stac_raster_bands.offset:
-        data_type = stac_raster_bands.data_type or data.dtype
+    if apply_offset and band_properties.offset:
+        data_type = band_properties.data_type or data.dtype
 
         # determine value range for the target data_type
         clip_min, clip_max = dtype_ranges[str(data_type)]
 
         # increase minimum clip value to avoid collission with nodata value
-        if clip_min == stac_raster_bands.nodata:
+        if clip_min == band_properties.nodata:
             clip_min += 1
 
         data[:] = (
             (
-                ((data * stac_raster_bands.scale) + stac_raster_bands.offset)
-                / stac_raster_bands.scale
+                ((data * band_properties.scale) + band_properties.offset)
+                / band_properties.scale
             )
             .round()
             .clip(clip_min, clip_max)

mapchete_eo/io/items.py

@@ -1,15 +1,15 @@
 import logging
-from typing import Any, List, Optional
+from typing import Any, List, Optional, Tuple, Union
 
 import numpy.ma as ma
 import pystac
+from mapchete.geometry import repair_antimeridian_geometry
 from mapchete.protocols import GridProtocol
 from mapchete.types import Bounds, NodataVals
 from rasterio.enums import Resampling
 from shapely.geometry import mapping, shape
 
 from mapchete_eo.exceptions import EmptyProductException
-from mapchete_eo.geometry import repair_antimeridian_geometry
 from mapchete_eo.io.assets import asset_to_np_array
 from mapchete_eo.types import BandLocation
 
@@ -56,7 +56,7 @@ def item_to_np_array(
     return out
 
 
-def expand_params(param, length):
+def expand_params(param: Any, length: int) -> List[Any]:
     """
     Expand parameters if they are not a list.
     """
@@ -69,7 +69,8 @@ def expand_params(param, length):
 
 def get_item_property(
     item: pystac.Item,
-    property: str,
+    property: Union[str, Tuple[str, ...]],
+    default: Any = None,
 ) -> Any:
     """
     Return item property.
@@ -104,26 +105,40 @@ def get_item_property(
     | ``collection``     | The collection ID of an Item's collection.             |
     +--------------------+--------------------------------------------------------+
     """
-    if property in ["year", "month", "day", "date", "datetime"]:
-        if item.datetime is None:
-            raise ValueError(
-                f"STAC item has no datetime attached, thus cannot get property {property}"
-            )
-        elif property == "date":
-            return item.datetime.date().isoformat()
-        elif property == "datetime":
-            return item.datetime
+
+    def _get_item_property(item: pystac.Item, property: str) -> Any:
+        if property == "id":
+            return item.id
+        elif property in ["year", "month", "day", "date", "datetime"]:
+            if item.datetime is None:  # pragma: no cover
+                raise ValueError(
+                    f"STAC item has no datetime attached, thus cannot get property {property}"
+                )
+            elif property == "date":
+                return item.datetime.date().isoformat()
+            elif property == "datetime":
+                return item.datetime
+            else:
+                return item.datetime.__getattribute__(property)
+        elif property == "collection":
+            return item.collection_id
+        elif property in item.properties:
+            return item.properties[property]
+        elif property in item.extra_fields:
+            return item.extra_fields[property]
+        elif property == "stac_extensions":
+            return item.stac_extensions
         else:
-            return item.datetime.__getattribute__(property)
-    elif property == "collection":
-        return item.collection_id
-    elif property in item.properties:
-        return item.properties[property]
-    elif property in item.extra_fields:
-        return item.extra_fields[property]
-    elif property == "stac_extensions":
-        return item.stac_extensions
+            raise KeyError
+
+    for prop in property if isinstance(property, tuple) else (property,):
+        try:
+            return _get_item_property(item, prop)
+        except KeyError:
+            pass
     else:
+        if default is not None:
+            return default
         raise KeyError(
             f"item {item.id} does not have property {property} in its datetime, properties "
             f"({', '.join(item.properties.keys())}) or extra_fields "

mapchete_eo/io/levelled_cubes.py

@@ -40,27 +40,50 @@ def read_levelled_cube_to_np_array(
     raise_empty: bool = True,
     out_dtype: DTypeLike = np.uint16,
     out_fill_value: NodataVal = 0,
+    read_mask: Optional[np.ndarray] = None,
 ) -> ma.MaskedArray:
     """
     Read products as slices into a cube by filling up nodata gaps with next slice.
+
+    If a read_mask is provided, only the pixels marked True are considered to be read.
     """
-    if len(products) == 0:
+    if len(products) == 0:  # pragma: no cover
         raise NoSourceProducts("no products to read")
-
     bands = assets or eo_bands
-    if bands is None:
+    if bands is None:  # pragma: no cover
         raise ValueError("either assets or eo_bands have to be set")
-
     out_shape = (target_height, len(bands), *grid.shape)
+
+    # 2D read_mask shape
+    if read_mask is None:
+        read_mask = np.ones(grid.shape, dtype=bool)
+    elif read_mask.ndim != 2:  # pragma: no cover
+        raise ValueError(
+            "read_mask must be 2-dimensional, not %s-dimensional",
+            read_mask.ndim,
+        )
     out: ma.MaskedArray = ma.masked_array(
-        data=np.zeros(out_shape, dtype=out_dtype),
-        mask=np.ones(out_shape, dtype=out_dtype),
+        data=np.full(out_shape, out_fill_value, dtype=out_dtype),
+        mask=np.ones(out_shape, dtype=bool),
         fill_value=out_fill_value,
     )
+
+    if not read_mask.any():
+        logger.debug("nothing to read")
+        return out
+
+    # extrude mask to match each layer
+    layer_read_mask = np.stack([read_mask for _ in bands])
+
+    def _cube_read_mask() -> np.ndarray:
+        # This is only needed for debug output, thus there is no need to materialize always
+        return np.stack([layer_read_mask for _ in range(target_height)])
+
     logger.debug(
-        "empty cube with shape %s has %s",
+        "empty cube with shape %s has %s and %s pixels to be filled",
         out.shape,
         pretty_bytes(out.size * out.itemsize),
+        _cube_read_mask().sum(),
     )
 
     logger.debug("sort products into slices ...")
@@ -76,25 +99,25 @@ def read_levelled_cube_to_np_array(
     slices_read_count, slices_skip_count = 0, 0
 
     # pick slices one by one
-    for slice_count, slice in enumerate(slices, 1):
+    for slice_count, slice_ in enumerate(slices, 1):
         # all filled up? let's get outta here!
         if not out.mask.any():
-            logger.debug("cube is full, quitting!")
+            logger.debug("cube has no pixels to be filled, quitting!")
             break
 
         # generate 2D mask of holes to be filled in output cube
-        cube_nodata_mask = out.mask.any(axis=0).any(axis=0)
+        cube_nodata_mask = np.logical_and(out.mask.any(axis=0).any(axis=0), read_mask)
 
         # read slice
         try:
             logger.debug(
                 "see if slice %s %s has some of the %s unmasked pixels for cube",
                 slice_count,
-                slice,
+                slice_,
                 cube_nodata_mask.sum(),
             )
-            with slice.cached():
-                slice_array = slice.read(
+            with slice_.cached():
+                slice_array = slice_.read(
                     merge_method=merge_method,
                     product_read_kwargs=dict(
                         product_read_kwargs,
@@ -104,17 +127,18 @@ def read_levelled_cube_to_np_array(
                         resampling=resampling,
                         nodatavals=nodatavals,
                         raise_empty=raise_empty,
-                        target_mask=~cube_nodata_mask.copy(),
+                        read_mask=cube_nodata_mask.copy(),
+                        out_dtype=out_dtype,
                     ),
                 )
             slices_read_count += 1
         except (EmptySliceException, CorruptedSlice) as exc:
-            logger.debug("skipped slice %s: %s", slice, str(exc))
+            logger.debug("skipped slice %s: %s", slice_, str(exc))
             slices_skip_count += 1
             continue
 
         # if slice was not empty, fill pixels into cube
-        logger.debug("add slice %s array to cube", slice)
+        logger.debug("add slice %s array to cube", slice_)
 
         # iterate through layers of cube
         for layer_index in range(target_height):
@@ -124,34 +148,35 @@ def read_levelled_cube_to_np_array(
                 continue
 
             # determine empty patches of current layer
-            empty_patches = out[layer_index].mask.copy()
-            pixels_for_layer = (~slice_array[empty_patches].mask).sum()
+            empty_patches = np.logical_and(out[layer_index].mask, layer_read_mask)
+            remaining_pixels_for_layer = (~slice_array[empty_patches].mask).sum()
 
             # when slice has nothing to offer for this layer, skip
-            if pixels_for_layer == 0:
+            if remaining_pixels_for_layer == 0:
                 logger.debug(
                     "layer %s: slice has no pixels for this layer, jump to next",
                     layer_index,
                 )
                 continue
 
+            # insert slice data into empty patches of layer
             logger.debug(
                 "layer %s: fill with %s pixels ...",
                 layer_index,
-                pixels_for_layer,
+                remaining_pixels_for_layer,
             )
-            # insert slice data into empty patches of layer
             out[layer_index][empty_patches] = slice_array[empty_patches]
-            masked_pixels = out[layer_index].mask.sum()
-            total_pixels = out[layer_index].size
-            percent_full = round(
-                100 * ((total_pixels - masked_pixels) / total_pixels), 2
-            )
+
+            # report on layer fill status
             logger.debug(
-                "layer %s: %s%% filled (%s empty pixels remaining)",
+                "layer %s: %s",
                 layer_index,
-                percent_full,
-                out[layer_index].mask.sum(),
+                _percent_full(
+                    remaining=np.logical_and(
+                        out[layer_index].mask, layer_read_mask
+                    ).sum(),
+                    total=layer_read_mask.sum(),
+                ),
             )
 
             # remove slice values which were just inserted for next layer
@@ -161,13 +186,13 @@ def read_levelled_cube_to_np_array(
                 logger.debug("slice fully inserted into cube, skipping")
                 break
 
-        masked_pixels = out.mask.sum()
-        total_pixels = out.size
-        percent_full = round(100 * ((total_pixels - masked_pixels) / total_pixels), 2)
+        # report on layer fill status
         logger.debug(
-            "cube is %s%% filled (%s empty pixels remaining)",
-            percent_full,
-            masked_pixels,
+            "cube is %s",
+            _percent_full(
+                remaining=np.logical_and(out.mask, _cube_read_mask()).sum(),
+                total=_cube_read_mask().sum(),
+            ),
         )
 
     logger.debug(
@@ -197,6 +222,7 @@ def read_levelled_cube_to_xarray(
     band_axis_name: str = "bands",
     x_axis_name: str = "x",
     y_axis_name: str = "y",
+    read_mask: Optional[np.ndarray] = None,
 ) -> xr.Dataset:
     """
     Read products as slices into a cube by filling up nodata gaps with next slice.
@@ -218,6 +244,7 @@ def read_levelled_cube_to_xarray(
             sort=sort,
             product_read_kwargs=product_read_kwargs,
             raise_empty=raise_empty,
+            read_mask=read_mask,
         ),
         slice_names=[f"layer-{ii}" for ii in range(target_height)],
         band_names=variables,
@@ -226,3 +253,7 @@ def read_levelled_cube_to_xarray(
         x_axis_name=x_axis_name,
         y_axis_name=y_axis_name,
     )
+
+
+def _percent_full(remaining: int, total: int, ndigits: int = 2) -> str:
+    return f"{round(100 * (total - remaining) / total, ndigits=ndigits)}% full ({remaining} remaining emtpy pixels)"

mapchete_eo/io/path.py

@@ -3,7 +3,7 @@ import logging
 from contextlib import contextmanager
 from enum import Enum
 from tempfile import TemporaryDirectory
-from typing import Generator
+from typing import Generator, Tuple, Union
 from xml.etree.ElementTree import Element, fromstring
 
 import fsspec
@@ -126,19 +126,30 @@ def cached_path(path: MPath, active: bool = True) -> Generator[MPath, None, None
 
 def asset_mpath(
     item: pystac.Item,
-    asset: str,
+    asset: Union[str, Tuple[str, ...]],
     fs: fsspec.AbstractFileSystem = None,
     absolute_path: bool = True,
 ) -> MPath:
     """Return MPath instance with asset href."""
 
-    try:
+    def _asset_mpath(
+        item: pystac.Item,
+        asset: str,
+        fs: fsspec.AbstractFileSystem = None,
+        absolute_path: bool = True,
+    ) -> MPath:
         asset_path = MPath(item.assets[asset].href, fs=fs)
-    except KeyError:
+        if absolute_path and not asset_path.is_absolute():
+            return MPath(item.get_self_href(), fs=fs).parent / asset_path
+        else:
+            return asset_path
+
+    for single_asset in asset if isinstance(asset, tuple) else (asset,):
+        try:
+            return _asset_mpath(item, single_asset, fs=fs, absolute_path=absolute_path)
+        except KeyError:
+            pass
+    else:
         raise AssetKeyError(
             f"{item.id} no asset named '{asset}' found in assets: {', '.join(item.assets.keys())}"
         )
-    if absolute_path and not asset_path.is_absolute():
-        return MPath(item.get_self_href(), fs=fs).parent / asset_path
-    else:
-        return asset_path

mapchete_eo/io/products.py

@@ -10,6 +10,7 @@ from typing import Any, Dict, Generator, Iterator, List, Optional, Sequence
 from mapchete import Timer
 import numpy as np
 import numpy.ma as ma
+from numpy.typing import DTypeLike
 import xarray as xr
 from mapchete.config import get_hash
 from mapchete.geometry import to_shape
@@ -28,7 +29,6 @@ from mapchete_eo.exceptions import (
     EmptyStackException,
     NoSourceProducts,
 )
-from mapchete_eo.io.items import get_item_property
 from mapchete_eo.protocols import EOProductProtocol
 from mapchete_eo.sort import SortMethodConfig
 from mapchete_eo.types import MergeMethod
@@ -49,11 +49,13 @@ def products_to_np_array(
     sort: Optional[SortMethodConfig] = None,
     product_read_kwargs: dict = {},
     raise_empty: bool = True,
+    out_dtype: Optional[DTypeLike] = None,
+    read_mask: Optional[np.ndarray] = None,
 ) -> ma.MaskedArray:
     """Read grid window of EOProducts and merge into a 4D xarray."""
     return ma.stack(
         [
-            to_masked_array(s)
+            to_masked_array(s, out_dtype=out_dtype)
             for s in generate_slice_dataarrays(
                 products=products,
                 assets=assets,
@@ -66,6 +68,7 @@ def products_to_np_array(
                 sort=sort,
                 product_read_kwargs=product_read_kwargs,
                 raise_empty=raise_empty,
+                read_mask=read_mask,
             )
         ]
     )
@@ -87,6 +90,7 @@ def products_to_xarray(
     sort: Optional[SortMethodConfig] = None,
     raise_empty: bool = True,
     product_read_kwargs: dict = {},
+    read_mask: Optional[np.ndarray] = None,
 ) -> xr.Dataset:
     """Read grid window of EOProducts and merge into a 4D xarray."""
     data_vars = [
@@ -103,6 +107,7 @@ def products_to_xarray(
             sort=sort,
             product_read_kwargs=product_read_kwargs,
             raise_empty=raise_empty,
+            read_mask=read_mask,
         )
     ]
     if merge_products_by and merge_products_by not in ["date", "datetime"]:
@@ -112,7 +117,8 @@ def products_to_xarray(
         coords = {
             slice_axis_name: list(
                 np.array(
-                    [product.item.datetime for product in products], dtype=np.datetime64
+                    [product.get_property("datetime") for product in products],
+                    dtype=np.datetime64,
                 )
             )
         }
@@ -126,7 +132,6 @@ class Slice:
     """Combine multiple products into one slice."""
 
     name: Any
-    properties: dict
     products: Sequence[EOProductProtocol]
     datetime: datetime
 
@@ -145,21 +150,13 @@ class Slice:
 
         # calculate mean datetime
         timestamps = [
-            product.item.datetime.timestamp()
+            product.get_property("datetime").timestamp()
             for product in self.products
-            if product.item.datetime
+            if product.get_property("datetime")
         ]
         mean_timestamp = sum(timestamps) / len(timestamps)
         self.datetime = datetime.fromtimestamp(mean_timestamp)
 
-        # generate combined properties
-        self.properties = {}
-        for key in self.products[0].item.properties.keys():
-            try:
-                self.properties[key] = self.get_property(key)
-            except ValueError:
-                self.properties[key] = None
-
     def __repr__(self) -> str:
         return f"<Slice {self.name} ({len(self.products)} products)>"
 
@@ -172,6 +169,17 @@ class Slice:
 
         raise EmptySliceException
 
+    @property
+    def properties(self) -> Dict[str, Any]:
+        # generate combined properties
+        properties: Dict[str, Any] = {}
+        for key in self.products[0].item.properties.keys():
+            try:
+                properties[key] = self.get_property(key)
+            except ValueError:
+                properties[key] = None
+        return properties
+
     @contextmanager
     def cached(self) -> Generator[Slice, None, None]:
         """Clear caches and run garbage collector when context manager is closed."""
@@ -194,12 +202,9 @@ class Slice:
         ValueError is raised.
         """
         # if set of value hashes has a length of 1, all values are the same
-        values = [
-            get_hash(get_item_property(product.item, property=property))
-            for product in self.products
-        ]
+        values = [get_hash(product.get_property(property)) for product in self.products]
         if len(set(values)) == 1:
-            return get_item_property(self.products[0].item, property=property)
+            return self.products[0].get_property(property)
 
         raise ValueError(
             f"cannot get unique property {property} from products {self.products}"
@@ -232,7 +237,7 @@ def products_to_slices(
             grouped[product.get_property(group_by_property)].append(product)
         slices = [Slice(key, products) for key, products in grouped.items()]
     else:
-        slices = [Slice(product.item.id, [product]) for product in products]
+        slices = [Slice(product.id, [product]) for product in products]
 
     # also check if slices is even a list, otherwise it will raise an error
     if sort and slices:
@@ -262,7 +267,7 @@ def merge_products(
                 try:
                     yield product.read_np_array(**product_read_kwargs)
                 except (AssetKeyError, CorruptedProduct) as exc:
-                    logger.debug("skip product %s because of %s", product.item.id, exc)
+                    logger.warning("skip product %s because of %s", product.id, exc)
         except StopIteration:
             return
 
@@ -280,7 +285,7 @@ def merge_products(
             out = product.read_np_array(**product_read_kwargs)
             break
         except (AssetKeyError, CorruptedProduct) as exc:
-            logger.debug("skip product %s because of %s", product.item.id, exc)
+            logger.warning("skip product %s because of %s", product.id, exc)
     else:
         # we cannot do anything here, as all products are broken
         raise CorruptedSlice("all products are broken here")
@@ -322,8 +327,11 @@ def merge_products(
         valid_arrays = [a for a in arrays if not ma.getmaskarray(a).all()]
 
         if valid_arrays:
-            stacked = ma.stack(valid_arrays, dtype=out.dtype)
-            out = stacked.mean(axis=0, dtype=out.dtype)
+            out_dtype = out.dtype
+            out_fill_value = out.fill_value
+            stacked = ma.stack(valid_arrays, dtype=out_dtype)
+            out = stacked.mean(axis=0, dtype=out_dtype).astype(out_dtype, copy=False)
+            out.set_fill_value(out_fill_value)
         else:
             # All arrays were fully masked — return fully masked output
             out = ma.masked_all(out.shape, dtype=out.dtype)
@@ -351,10 +359,12 @@ def generate_slice_dataarrays(
     sort: Optional[SortMethodConfig] = None,
     product_read_kwargs: dict = {},
     raise_empty: bool = True,
+    read_mask: Optional[np.ndarray] = None,
 ) -> Iterator[xr.DataArray]:
     """
     Yield products or merged products into slices as DataArrays.
     """
+
     if len(products) == 0:
         raise NoSourceProducts("no products to read")
 
@@ -367,7 +377,6 @@ def generate_slice_dataarrays(
     slices = products_to_slices(
         products, group_by_property=merge_products_by, sort=sort
     )
-
     logger.debug(
         "reading %s products in %s groups...",
         len(products),
@@ -396,6 +405,7 @@ def generate_slice_dataarrays(
                             resampling=resampling,
                             nodatavals=nodatavals,
                             raise_empty=raise_empty,
+                            read_mask=read_mask,
                         ),
                         raise_empty=raise_empty,
                     ),
@@ -406,8 +416,8 @@ def generate_slice_dataarrays(
                 )
             # if at least one slice can be yielded, the stack is not empty
             stack_empty = False
-        except (EmptySliceException, CorruptedSlice):
-            pass
+        except (EmptySliceException, CorruptedSlice) as exception:
+            logger.warning(exception)
 
     if stack_empty:
         raise EmptyStackException("all slices are empty")

mapchete_eo/platforms/sentinel2/__init__.py

@@ -4,7 +4,7 @@ from mapchete_eo.platforms.sentinel2.driver import (
     Sentinel2Cube,
     Sentinel2CubeGroup,
 )
-from mapchete_eo.platforms.sentinel2.metadata_parser import S2Metadata
+from mapchete_eo.platforms.sentinel2.metadata_parser.s2metadata import S2Metadata
 from mapchete_eo.platforms.sentinel2.product import S2Product
 
 __all__ = [