mapchete-eo 2026.2.0__py3-none-any.whl

mapchete_eo/io/products.py

@@ -0,0 +1,423 @@
+from __future__ import annotations
+
+from contextlib import contextmanager
+import logging
+from collections import defaultdict
+from datetime import datetime
+import gc
+from typing import Any, Dict, Generator, Iterator, List, Optional, Sequence
+
+from mapchete import Timer
+import numpy as np
+import numpy.ma as ma
+from numpy.typing import DTypeLike
+import xarray as xr
+from mapchete.config import get_hash
+from mapchete.geometry import to_shape
+from mapchete.protocols import GridProtocol
+from mapchete.types import NodataVals
+from rasterio.enums import Resampling
+from shapely.geometry import mapping
+from shapely.ops import unary_union
+
+from mapchete_eo.array.convert import to_dataarray, to_masked_array
+from mapchete_eo.exceptions import (
+    AssetKeyError,
+    CorruptedProduct,
+    CorruptedSlice,
+    EmptySliceException,
+    EmptyStackException,
+    NoSourceProducts,
+)
+from mapchete_eo.protocols import EOProductProtocol
+from mapchete_eo.sort import SortMethodConfig
+from mapchete_eo.types import MergeMethod
+
+
+logger = logging.getLogger(__name__)
+
+
+def products_to_np_array(
+    products: List[EOProductProtocol],
+    assets: Optional[List[str]] = None,
+    eo_bands: Optional[List[str]] = None,
+    grid: Optional[GridProtocol] = None,
+    resampling: Resampling = Resampling.nearest,
+    nodatavals: NodataVals = None,
+    merge_products_by: Optional[str] = None,
+    merge_method: MergeMethod = MergeMethod.first,
+    sort: Optional[SortMethodConfig] = None,
+    product_read_kwargs: dict = {},
+    raise_empty: bool = True,
+    out_dtype: Optional[DTypeLike] = None,
+    read_mask: Optional[np.ndarray] = None,
+) -> ma.MaskedArray:
+    """Read grid window of EOProducts and merge into a 4D xarray."""
+    return ma.stack(
+        [
+            to_masked_array(s, out_dtype=out_dtype)
+            for s in generate_slice_dataarrays(
+                products=products,
+                assets=assets,
+                eo_bands=eo_bands,
+                grid=grid,
+                resampling=resampling,
+                nodatavals=nodatavals,
+                merge_products_by=merge_products_by,
+                merge_method=merge_method,
+                sort=sort,
+                product_read_kwargs=product_read_kwargs,
+                raise_empty=raise_empty,
+                read_mask=read_mask,
+            )
+        ]
+    )
+
+
+def products_to_xarray(
+    products: List[EOProductProtocol],
+    assets: Optional[List[str]] = None,
+    eo_bands: Optional[List[str]] = None,
+    grid: Optional[GridProtocol] = None,
+    resampling: Resampling = Resampling.nearest,
+    nodatavals: NodataVals = None,
+    slice_axis_name: str = "time",
+    band_axis_name: str = "bands",
+    x_axis_name: str = "x",
+    y_axis_name: str = "y",
+    merge_products_by: Optional[str] = None,
+    merge_method: MergeMethod = MergeMethod.first,
+    sort: Optional[SortMethodConfig] = None,
+    raise_empty: bool = True,
+    product_read_kwargs: dict = {},
+    read_mask: Optional[np.ndarray] = None,
+) -> xr.Dataset:
+    """Read grid window of EOProducts and merge into a 4D xarray."""
+    data_vars = [
+        s
+        for s in generate_slice_dataarrays(
+            products=products,
+            assets=assets,
+            eo_bands=eo_bands,
+            grid=grid,
+            resampling=resampling,
+            nodatavals=nodatavals,
+            merge_products_by=merge_products_by,
+            merge_method=merge_method,
+            sort=sort,
+            product_read_kwargs=product_read_kwargs,
+            raise_empty=raise_empty,
+            read_mask=read_mask,
+        )
+    ]
+    if merge_products_by and merge_products_by not in ["date", "datetime"]:
+        coords = {merge_products_by: [s.name for s in data_vars]}
+        slice_axis_name = merge_products_by
+    else:
+        coords = {
+            slice_axis_name: list(
+                np.array(
+                    [product.get_property("datetime") for product in products],
+                    dtype=np.datetime64,
+                )
+            )
+        }
+    return xr.Dataset(
+        data_vars={s.name: s for s in data_vars},
+        coords=coords,
+    ).transpose(slice_axis_name, band_axis_name, x_axis_name, y_axis_name)
+
+
+class Slice:
+    """Combine multiple products into one slice."""
+
+    name: Any
+    products: Sequence[EOProductProtocol]
+    datetime: datetime
+
+    def __init__(
+        self,
+        name: Any,
+        products: Sequence[EOProductProtocol],
+    ):
+        self.name = name
+
+        # a Slice can only be valid if it contains one or more products
+        if products:
+            self.products = products
+        else:  # pragma: no cover
+            raise ValueError("at least one product must be provided.")
+
+        # calculate mean datetime
+        timestamps = [
+            product.get_property("datetime").timestamp()
+            for product in self.products
+            if product.get_property("datetime")
+        ]
+        mean_timestamp = sum(timestamps) / len(timestamps)
+        self.datetime = datetime.fromtimestamp(mean_timestamp)
+
+    def __repr__(self) -> str:
+        return f"<Slice {self.name} ({len(self.products)} products)>"
+
+    @property
+    def __geom_interface__(self) -> Dict:
+        if self.products:
+            return mapping(
+                unary_union([to_shape(product) for product in self.products])
+            )
+
+        raise EmptySliceException
+
+    @property
+    def properties(self) -> Dict[str, Any]:
+        # generate combined properties
+        properties: Dict[str, Any] = {}
+        for key in self.products[0].item.properties.keys():
+            try:
+                properties[key] = self.get_property(key)
+            except ValueError:
+                properties[key] = None
+        return properties
+
+    @contextmanager
+    def cached(self) -> Generator[Slice, None, None]:
+        """Clear caches and run garbage collector when context manager is closed."""
+        yield self
+        with Timer() as tt:
+            self.clear_cached_data()
+            gc.collect()
+        logger.debug("Slice cache cleared and garbage collected in %s", tt)
+
+    def clear_cached_data(self):
+        logger.debug("clear caches of all products in slice")
+        for product in self.products:
+            product.clear_cached_data()
+
+    def get_property(self, property: str) -> Any:
+        """
+        Return merged property over all products.
+
+        If property values are the same over all products, it will be returned. Otherwise a
+        ValueError is raised.
+        """
+        # if set of value hashes has a length of 1, all values are the same
+        values = [get_hash(product.get_property(property)) for product in self.products]
+        if len(set(values)) == 1:
+            return self.products[0].get_property(property)
+
+        raise ValueError(
+            f"cannot get unique property {property} from products {self.products}"
+        )
+
+    def read(
+        self,
+        merge_method: MergeMethod = MergeMethod.first,
+        product_read_kwargs: dict = {},
+        raise_empty: bool = True,
+    ) -> ma.MaskedArray:
+        logger.debug("Slice: read from %s products", len(self.products))
+        return merge_products(
+            products=self.products,
+            merge_method=merge_method,
+            product_read_kwargs=product_read_kwargs,
+            raise_empty=raise_empty,
+        )
+
+
+def products_to_slices(
+    products: List[EOProductProtocol],
+    group_by_property: Optional[str] = None,
+    sort: Optional[SortMethodConfig] = None,
+) -> List[Slice]:
+    """Group products per given property into Slice objects and optionally sort slices."""
+    if group_by_property:
+        grouped = defaultdict(list)
+        for product in products:
+            grouped[product.get_property(group_by_property)].append(product)
+        slices = [Slice(key, products) for key, products in grouped.items()]
+    else:
+        slices = [Slice(product.id, [product]) for product in products]
+
+    # also check if slices is even a list, otherwise it will raise an error
+    if sort and slices:
+        sort_dict = sort.model_dump()
+        func = sort_dict.pop("func")
+        slices = func(slices, **sort_dict)
+
+    return slices
+
+
+def merge_products(
+    products: Sequence[EOProductProtocol],
+    merge_method: MergeMethod = MergeMethod.first,
+    product_read_kwargs: dict = {},
+    raise_empty: bool = True,
+) -> ma.MaskedArray:
+    """
+    Merge given products into one array.
+    """
+
+    def read_remaining_valid_products(
+        products_iter: Iterator[EOProductProtocol], product_read_kwargs: dict
+    ) -> Generator[ma.MaskedArray, None, None]:
+        """Yields and reads remaining products from iterator while discarding corrupt products."""
+        try:
+            for product in products_iter:
+                try:
+                    yield product.read_np_array(**product_read_kwargs)
+                except (AssetKeyError, CorruptedProduct) as exc:
+                    logger.warning("skip product %s because of %s", product.id, exc)
+        except StopIteration:
+            return
+
+    if len(products) == 0:  # pragma: no cover
+        raise NoSourceProducts("no products to merge")
+
+    # we need to deactivate raising the EmptyProductException
+    product_read_kwargs.update(raise_empty=False)
+
+    products_iter = iter(products)
+
+    # read first valid product
+    for product in products_iter:
+        try:
+            out = product.read_np_array(**product_read_kwargs)
+            break
+        except (AssetKeyError, CorruptedProduct) as exc:
+            logger.warning("skip product %s because of %s", product.id, exc)
+    else:
+        # we cannot do anything here, as all products are broken
+        raise CorruptedSlice("all products are broken here")
+
+    # fill in gaps sequentially, product by product
+    if merge_method == MergeMethod.first:
+        for new in read_remaining_valid_products(products_iter, product_read_kwargs):
+            masked = out.mask
+            # Update values at masked locations
+            out[masked] = new[masked]
+            # Update mask at masked locations (e.g., unmask them)
+            out.mask[masked] = new.mask[masked]
+            # if whole output array is filled, there is no point in reading more data
+            if not out.mask.any():
+                return out
+
+    # read all and average
+    elif merge_method == MergeMethod.average:
+
+        def _generate_arrays(
+            first_product_array: ma.MaskedArray,
+            remaining_product_arrays: Generator[ma.MaskedArray, None, None],
+        ) -> Generator[ma.MaskedArray, None, None]:
+            """Yield all available product arrays."""
+            yield first_product_array
+            yield from remaining_product_arrays
+
+        # explicitly specify dtype to avoid casting of integer arrays to floats
+        # during mean conversion:
+        # https://numpy.org/doc/stable/reference/generated/numpy.mean.html#numpy.mean
+        arrays = list(
+            _generate_arrays(
+                out,
+                read_remaining_valid_products(products_iter, product_read_kwargs),
+            )
+        )
+
+        # Filter out arrays that are entirely masked
+        valid_arrays = [a for a in arrays if not ma.getmaskarray(a).all()]
+
+        if valid_arrays:
+            out_dtype = out.dtype
+            out_fill_value = out.fill_value
+            stacked = ma.stack(valid_arrays, dtype=out_dtype)
+            out = stacked.mean(axis=0, dtype=out_dtype).astype(out_dtype, copy=False)
+            out.set_fill_value(out_fill_value)
+        else:
+            # All arrays were fully masked — return fully masked output
+            out = ma.masked_all(out.shape, dtype=out.dtype)
+
+    else:  # pragma: no cover
+        raise NotImplementedError(f"unknown merge method: {merge_method}")
+
+    if raise_empty and out.mask.all():
+        raise EmptySliceException(
+            f"slice is empty after combining {len(products)} products"
+        )
+
+    return out
+
+
+def generate_slice_dataarrays(
+    products: List[EOProductProtocol],
+    assets: Optional[List[str]] = None,
+    eo_bands: Optional[List[str]] = None,
+    grid: Optional[GridProtocol] = None,
+    resampling: Resampling = Resampling.nearest,
+    nodatavals: NodataVals = None,
+    merge_products_by: Optional[str] = None,
+    merge_method: MergeMethod = MergeMethod.first,
+    sort: Optional[SortMethodConfig] = None,
+    product_read_kwargs: dict = {},
+    raise_empty: bool = True,
+    read_mask: Optional[np.ndarray] = None,
+) -> Iterator[xr.DataArray]:
+    """
+    Yield products or merged products into slices as DataArrays.
+    """
+
+    if len(products) == 0:
+        raise NoSourceProducts("no products to read")
+
+    stack_empty = True
+    assets = assets or []
+    eo_bands = eo_bands or []
+    variables = assets or eo_bands
+
+    # group products into slices and sort slices if configured
+    slices = products_to_slices(
+        products, group_by_property=merge_products_by, sort=sort
+    )
+    logger.debug(
+        "reading %s products in %s groups...",
+        len(products),
+        len(slices),
+    )
+    if isinstance(nodatavals, list):
+        nodataval = nodatavals[0]
+    elif isinstance(nodatavals, float):
+        nodataval = nodatavals
+    else:
+        nodataval = nodatavals
+    for slice in slices:
+        try:
+            # if merge_products_by is none, each slice contains just one product
+            # so nothing will have to be merged anyways
+            with slice.cached():
+                yield to_dataarray(
+                    merge_products(
+                        products=slice.products,
+                        merge_method=merge_method,
+                        product_read_kwargs=dict(
+                            product_read_kwargs,
+                            assets=assets,
+                            eo_bands=eo_bands,
+                            grid=grid,
+                            resampling=resampling,
+                            nodatavals=nodatavals,
+                            raise_empty=raise_empty,
+                            read_mask=read_mask,
+                        ),
+                        raise_empty=raise_empty,
+                    ),
+                    nodataval=nodataval,
+                    name=slice.name,
+                    band_names=variables,
+                    attrs=slice.properties,
+                )
+            # if at least one slice can be yielded, the stack is not empty
+            stack_empty = False
+        except (EmptySliceException, CorruptedSlice) as exception:
+            logger.warning(exception)
+
+    if stack_empty:
+        raise EmptyStackException("all slices are empty")

mapchete_eo/io/profiles.py

@@ -0,0 +1,45 @@
+from rasterio.profiles import Profile
+
+
+class COGDeflateProfile(Profile):
+    """Standard COG profile."""
+
+    defaults = {
+        "driver": "COG",
+        "tiled": True,
+        "blockxsize": 512,
+        "blockysize": 512,
+        "compress": "DEFLATE",
+    }
+
+
+class JP2LossyProfile(Profile):
+    """Very lossy JP2 profile used for low size test data."""
+
+    defaults = {
+        "driver": "JP2OpenJPEG",
+        "tiled": True,
+        "blockxsize": 512,
+        "blockysize": 512,
+        "quality": 50,
+    }
+
+
+class JP2LosslessProfile(Profile):
+    """Lossless JP2 profile used for lower size data."""
+
+    defaults = {
+        "driver": "JP2OpenJPEG",
+        "tiled": True,
+        "blockxsize": 512,
+        "blockysize": 512,
+        "quality": 100,
+        "reversible": True,
+    }
+
+
+rio_profiles = {
+    "cog_deflate": COGDeflateProfile(),
+    "jp2_lossy": JP2LossyProfile(),
+    "jp2_lossless": JP2LosslessProfile(),
+}

mapchete_eo/platforms/sentinel2/__init__.py

@@ -0,0 +1,17 @@
+from mapchete_eo.platforms.sentinel2.driver import (
+    METADATA,
+    InputData,
+    Sentinel2Cube,
+    Sentinel2CubeGroup,
+)
+from mapchete_eo.platforms.sentinel2.metadata_parser.s2metadata import S2Metadata
+from mapchete_eo.platforms.sentinel2.product import S2Product
+
+__all__ = [
+    "S2Metadata",
+    "METADATA",
+    "InputData",
+    "Sentinel2Cube",
+    "Sentinel2CubeGroup",
+    "S2Product",
+]

mapchete_eo/platforms/sentinel2/_mapper_registry.py

@@ -0,0 +1,89 @@
+from typing import List, Callable, Dict, Any, Optional
+
+from pystac import Item
+
+from mapchete_eo.platforms.sentinel2.metadata_parser.s2metadata import S2Metadata
+from mapchete_eo.platforms.sentinel2.types import DataArchive, MetadataArchive
+
+
+# decorators for mapper functions using the registry pattern #
+##############################################################
+ID_MAPPER_REGISTRY: Dict[Any, Callable[[Item], Item]] = {}
+STAC_METADATA_MAPPER_REGISTRY: Dict[Any, Callable[[Item], Item]] = {}
+S2METADATA_MAPPER_REGISTRY: Dict[Any, Callable[[Item], S2Metadata]] = {}
+
+MAPPER_REGISTRIES: Dict[str, Any] = {
+    "ID": ID_MAPPER_REGISTRY,
+    "STAC metadata": STAC_METADATA_MAPPER_REGISTRY,
+    "S2Metadata": S2METADATA_MAPPER_REGISTRY,
+}
+
+
+def _register_func(registry: Dict[str, Callable], key: Any, func: Callable):
+    if key in registry:
+        raise ValueError(f"{key} already registered in {registry}")
+    registry[key] = func
+
+
+def maps_item_id(from_collections: List[str]):
+    """
+    Decorator registering mapper to common ID.
+    """
+
+    def decorator(func):
+        # Use a tuple of the metadata as the key
+        # key = (path_type, version)
+        for collection in from_collections:
+            _register_func(registry=ID_MAPPER_REGISTRY, key=collection, func=func)
+        return func
+
+    return decorator
+
+
+def maps_stac_metadata(
+    from_collections: List[str], to_data_archives: Optional[List[DataArchive]] = None
+):
+    """
+    Decorator registering STAC metadata mapper.
+    """
+
+    def decorator(func):
+        # Use a tuple of the metadata as the key
+        for collection in from_collections:
+            if to_data_archives:
+                for data_archive in to_data_archives:
+                    _register_func(
+                        registry=STAC_METADATA_MAPPER_REGISTRY,
+                        key=(collection, data_archive),
+                        func=func,
+                    )
+            else:
+                _register_func(
+                    registry=STAC_METADATA_MAPPER_REGISTRY,
+                    key=collection,
+                    func=func,
+                )
+        return func
+
+    return decorator
+
+
+def creates_s2metadata(
+    from_collections: List[str], to_metadata_archives: List[MetadataArchive]
+):
+    """
+    Decorator registering S2Metadata creator.
+    """
+
+    def decorator(func):
+        # Use a tuple of the metadata as the key
+        for collection in from_collections:
+            for metadata_archive in to_metadata_archives:
+                _register_func(
+                    registry=S2METADATA_MAPPER_REGISTRY,
+                    key=(collection, metadata_archive),
+                    func=func,
+                )
+        return func
+
+    return decorator

mapchete_eo/platforms/sentinel2/bandpass_adjustment.py

@@ -0,0 +1,104 @@
+from enum import Enum
+from typing import NamedTuple
+import numpy as np
+import numpy.ma as ma
+from numpy.typing import DTypeLike
+
+from pystac import Item
+
+from mapchete_eo.platforms.sentinel2.types import L2ABand
+
+
+class BandpassAdjustment(NamedTuple):
+    slope: float
+    intercept: float
+
+
+# Bandpass Adjustment for Sentinel-2
+# Try using HLS bandpass adjustmets
+# https://hls.gsfc.nasa.gov/algorithms/bandpass-adjustment/
+# https://lpdaac.usgs.gov/documents/1698/HLS_User_Guide_V2.pdf
+# These are for Sentinel-2B bandpass adjustment; fisrt is slope second is intercept
+# out_band = band * slope + intercept
+# B1	0.996	0.002
+# B2	1.001	-0.002
+# B3	0.999	0.001
+# B4	1.001	-0.003
+# B5	0.998	0.004
+# B6	0.997	0.005
+# B7	1.000	0.000
+# B8	0.999	0.001
+# B8A	0.998	0.004
+# B9	0.996	0.006
+# B10	1.001	-0.001  B10 is not present in Sentinel-2 L2A products ommited in params below
+# B11	0.997	0.002
+# B12	0.998	0.003
+
+
+class L2AS2ABandpassAdjustmentParams(Enum):
+    B01 = BandpassAdjustment(0.9959, -0.0002)
+    B02 = BandpassAdjustment(0.9778, -0.004)
+    B03 = BandpassAdjustment(1.0053, -0.0009)
+    B04 = BandpassAdjustment(0.9765, 0.0009)
+    B05 = BandpassAdjustment(1.0, 0.0)
+    B06 = BandpassAdjustment(1.0, 0.0)
+    B07 = BandpassAdjustment(1.0, 0.0)
+    B08 = BandpassAdjustment(0.9983, -0.0001)
+    B8A = BandpassAdjustment(0.9983, -0.0001)
+    B09 = BandpassAdjustment(1.0, 0.0)
+    B11 = BandpassAdjustment(0.9987, -0.0011)
+    B12 = BandpassAdjustment(1.003, -0.0012)
+
+
+class L2AS2BBandpassAdjustmentParams(Enum):
+    B01 = BandpassAdjustment(0.9959, -0.0002)
+    B02 = BandpassAdjustment(0.9778, -0.004)
+    B03 = BandpassAdjustment(1.0075, -0.0008)
+    B04 = BandpassAdjustment(0.9761, 0.001)
+    B05 = BandpassAdjustment(0.998, 0.004)
+    B06 = BandpassAdjustment(0.997, 0.005)
+    B07 = BandpassAdjustment(1.000, 0.000)
+    B08 = BandpassAdjustment(0.9966, 0.000)
+    B8A = BandpassAdjustment(0.9966, 0.000)
+    B09 = BandpassAdjustment(0.996, 0.006)
+    B11 = BandpassAdjustment(1.000, -0.0003)
+    B12 = BandpassAdjustment(0.9867, 0.0004)
+
+
+def item_to_params(
+    sentinel2_item: Item,
+    l2a_band: L2ABand,
+) -> BandpassAdjustment:
+    if sentinel2_item.properties["platform"].lower() == "sentinel-2a":
+        return L2AS2ABandpassAdjustmentParams[l2a_band.name].value
+    elif sentinel2_item.properties["platform"].lower() == "sentinel-2b":
+        return L2AS2BBandpassAdjustmentParams[l2a_band.name].value
+    else:
+        raise TypeError(
+            f"cannot determine Sentinel-2 platform from pystac.Item: {sentinel2_item}"
+        )
+
+
+def apply_bandpass_adjustment(
+    band_arr: ma.MaskedArray,
+    item: Item,
+    l2a_band: L2ABand,
+    computing_dtype: DTypeLike = np.float32,
+    out_dtype: DTypeLike = np.uint16,
+) -> ma.MaskedArray:
+    params = item_to_params(item, l2a_band)
+    return ma.MaskedArray(
+        data=(
+            np.clip(
+                band_arr.astype(computing_dtype, copy=False) / 10000 * params.slope
+                + params.intercept,
+                0,
+                1,
+            )
+            * 10000
+        )
+        .astype(out_dtype, copy=False)
+        .data,
+        mask=band_arr.mask,
+        fill_value=band_arr.fill_value,
+    )

mapchete_eo/platforms/sentinel2/brdf/__init__.py

@@ -0,0 +1,8 @@
+from mapchete_eo.platforms.sentinel2.brdf.correction import (
+    correction_values,
+    apply_correction,
+)
+
+from mapchete_eo.platforms.sentinel2.brdf.models import get_model
+
+__all__ = ["correction_values", "apply_correction", "get_model"]