mapchete-eo 2026.2.0__py3-none-any.whl

mapchete_eo/platforms/sentinel2/source.py

@@ -0,0 +1,114 @@
+from __future__ import annotations
+
+from typing import Optional, List, Callable, Dict, Any, Union
+import warnings
+
+from pydantic import model_validator
+from pystac import Item
+
+from mapchete_eo.platforms.sentinel2.metadata_parser.s2metadata import S2Metadata
+from mapchete_eo.source import Source
+from mapchete_eo.platforms.sentinel2.preconfigured_sources import (
+    DEPRECATED_ARCHIVES,
+    KNOWN_SOURCES,
+)
+from mapchete_eo.platforms.sentinel2.types import (
+    DataArchive,
+    MetadataArchive,
+)
+from mapchete_eo.platforms.sentinel2._mapper_registry import MAPPER_REGISTRIES
+
+
+def known_collection_to_url(collection: str) -> str:
+    if collection in KNOWN_SOURCES:
+        return KNOWN_SOURCES[collection]["collection"]
+    return collection
+
+
+class Sentinel2Source(Source):
+    """All information required to consume Sentinel-2 products."""
+
+    # extends base model with those properties
+    data_archive: Optional[DataArchive] = None
+    metadata_archive: MetadataArchive = "roda"
+
+    @property
+    def item_modifier_funcs(self) -> List[Callable]:
+        return [
+            func
+            for func in (self.get_id_mapper(), *self.get_stac_metadata_mappers())
+            if func is not None
+        ]
+
+    @model_validator(mode="before")
+    def determine_data_source(cls, values: Dict[str, Any]) -> Dict[str, Any]:
+        """Handles short names of sources."""
+        if isinstance(values, str):
+            values = dict(collection=values)
+        collection = values.get("collection", None)
+        if collection in KNOWN_SOURCES:
+            values.update(KNOWN_SOURCES[collection])
+        elif collection in DEPRECATED_ARCHIVES:
+            warnings.warn(
+                f"deprecated archive '{collection}' found",
+                category=DeprecationWarning,
+                stacklevel=2,
+            )
+            values.update(DEPRECATED_ARCHIVES[collection])
+        return values
+
+    @model_validator(mode="after")
+    def verify_mappers(self) -> Sentinel2Source:
+        # make sure all required mappers are registered
+        self.get_id_mapper()
+        self.get_stac_metadata_mappers()
+        self.get_s2metadata_mapper()
+        return self
+
+    def get_id_mapper(self) -> Union[Callable[[Item], Item], None]:
+        if self.catalog_type == "static":
+            return None
+        for key in MAPPER_REGISTRIES["ID"].keys():
+            if self.collection == known_collection_to_url(key):
+                return MAPPER_REGISTRIES["ID"][key]
+        else:
+            raise ValueError(f"no ID mapper for {self.collection} found")
+
+    def get_stac_metadata_mappers(self) -> List[Callable[[Item], Item]]:
+        """Find mapper function.
+
+        A mapper function must be provided if a custom data_archive was configured.
+        """
+        mappers: List[Callable] = []
+        if self.catalog_type == "static":
+            return mappers
+        for key in MAPPER_REGISTRIES["STAC metadata"]:
+            if isinstance(key, tuple):
+                collection, data_archive = key
+                if (
+                    self.collection == known_collection_to_url(collection)
+                    and data_archive == self.data_archive
+                ):
+                    mappers.append(MAPPER_REGISTRIES["STAC metadata"][key])
+            elif self.collection == known_collection_to_url(key):
+                mappers.append(MAPPER_REGISTRIES["STAC metadata"][key])
+        if mappers or self.data_archive is None:
+            return mappers
+        raise ValueError(
+            f"no STAC metadata mapper from {self.collection} to {self.data_archive} found"
+        )
+
+    def get_s2metadata_mapper(self) -> Union[Callable[[Item], S2Metadata], None]:
+        if self.catalog_type == "static" or self.metadata_archive is None:
+            return None
+        for key in MAPPER_REGISTRIES["S2Metadata"]:
+            collection, metadata_archive = key
+            if (
+                self.collection == known_collection_to_url(collection)
+                and metadata_archive == self.metadata_archive
+            ):
+                return MAPPER_REGISTRIES["S2Metadata"][key]
+        else:
+            raise ValueError(
+                f"no S2Metadata mapper from {self.collection} to {self.metadata_archive} found"
+            )

mapchete_eo/platforms/sentinel2/types.py

@@ -0,0 +1,114 @@
+from enum import Enum
+from typing import Literal
+
+Resolution = Enum(
+    "Resolution",
+    {
+        "original": None,
+        "10m": 10,
+        "20m": 20,
+        "60m": 60,
+        "120m": 120,
+    },
+)
+
+
+ProductQIMaskResolution = Enum(
+    "ProductQIMaskResolution",
+    {
+        "20m": 20,
+        "60m": 60,
+    },
+)
+
+
+class CloudType(str, Enum):
+    """Available cloud types in masks."""
+
+    opaque = "opaque"
+    cirrus = "cirrus"
+    all = "all"
+
+
+class ClassificationBandIndex(int, Enum):
+    """Band index used for classification masks."""
+
+    opaque = 1
+    cirrus = 2
+    # this is only available since PB 04.00
+    snow_ice = 3
+
+
+class L2ABand(int, Enum):
+    """Mapping between band identifier and metadata internal band index."""
+
+    B01 = 0
+    B02 = 1
+    B03 = 2
+    B04 = 3
+    B05 = 4
+    B06 = 5
+    B07 = 6
+    B08 = 7
+    B8A = 8
+    B09 = 9
+    B10 = 10
+    B11 = 11
+    B12 = 12
+
+
+class ProcessingLevel(Enum):
+    """Available processing levels of Sentinel-2."""
+
+    level1c = "L1C"
+    level2a = "L2A"
+
+
+class ProductQI(str, Enum):
+    """Product specific quality indicators."""
+
+    classification = "classification"
+    cloud_probability = "cloud_probability"
+    snow_probability = "snow_probability"
+
+
+class BandQI(str, Enum):
+    """Band specific quality indicators."""
+
+    detector_footprints = "detector_footprints"
+    technical_quality = "technical_quality"
+    # the following masks are deprecated:
+    # nodata = "nodata"
+    # defect = "defect"
+    # saturated = "saturated"
+
+
+class SunAngle(str, Enum):
+    zenith = "Zenith"
+    azimuth = "Azimuth"
+
+
+class ViewAngle(str, Enum):
+    zenith = "Zenith"
+    azimuth = "Azimuth"
+
+
+class SceneClassification(int, Enum):
+    """Mapping of pixel values to class in SCL bands."""
+
+    nodata = 0
+    saturated_or_defected = 1
+    dark_area_pixels = 2
+    cloud_shadows = 3
+    vegetation = 4
+    not_vegetated = 5
+    water = 6
+    unclassified = 7
+    cloud_medium_probability = 8
+    cloud_high_probability = 9
+    thin_cirrus = 10
+    snow = 11
+
+
+DataArchive = Literal["AWSCOG", "AWSJP2"]
+MetadataArchive = Literal["roda", "CDSE"]

mapchete_eo/processes/config.py

@@ -0,0 +1,51 @@
+from __future__ import annotations
+
+from typing import Dict, Optional, Tuple, Union
+
+from pydantic import BaseModel
+
+
+class SmoothConfig(BaseModel):
+    radius: Optional[int] = 1
+    smooth_more: Optional[bool] = False
+
+    @staticmethod
+    def parse(inp: Union[SmoothConfig, dict]) -> SmoothConfig:
+        if isinstance(inp, SmoothConfig):
+            return inp
+        elif isinstance(inp, dict):
+            return SmoothConfig(**inp)
+        else:
+            raise TypeError(f"cannot parse SmoothConfig from {inp}")
+
+
+class RGBCompositeConfig(BaseModel):
+    red: Tuple[int, int] = (0, 2300)
+    green: Tuple[int, int] = (0, 2300)
+    blue: Tuple[int, int] = (0, 2300)
+    gamma: float = 1.15
+    saturation: float = 1.3
+    clahe_flag: bool = True
+    clahe_clip_limit: float = 3.2
+    clahe_tile_grid_size: Tuple[int, int] = (32, 32)
+    sigmoidal_flag: bool = False
+    sigmoidal_contrast: int = 0
+    sigmoidal_bias: float = 0.0
+    fuzzy_radius: Optional[int] = 0
+    sharpen: Optional[bool] = False
+    smooth: Optional[bool] = False
+    smooth_config: SmoothConfig = SmoothConfig()
+    smooth_water: Optional[bool] = False
+    smooth_water_config: SmoothConfig = SmoothConfig(radius=6, smooth_more=True)
+    smooth_water_ndwi_threshold: float = 0.2
+    calculations_dtype: str = "float16"
+
+    @staticmethod
+    def parse(inp: Union[RGBCompositeConfig, Dict]) -> RGBCompositeConfig:
+        if isinstance(inp, RGBCompositeConfig):
+            return inp
+        elif isinstance(inp, dict):
+            smooth_config = SmoothConfig.parse(inp.pop("smooth_config", {}))
+            return RGBCompositeConfig(smooth_config=smooth_config, **inp)
+        else:
+            raise TypeError(f"cannot parse RGBCompositeConfig from {inp}")

mapchete_eo/processes/dtype_scale.py

@@ -0,0 +1,112 @@
+import logging
+from typing import Optional
+
+from mapchete import MapcheteProcess, RasterInput
+import numpy as np
+import numpy.ma as ma
+from mapchete.errors import MapcheteNodataTile
+from mapchete.types import NodataVal
+
+from mapchete_eo.exceptions import EmptyStackException
+from mapchete_eo.image_operations import dtype_scale
+
+logger = logging.getLogger(__name__)
+
+
+def execute(
+    mp: MapcheteProcess,
+    inp: RasterInput,
+    bands: list = [1, 2, 3, 4],
+    resampling: str = "nearest",
+    matching_method: Optional[str] = "gdal",
+    matching_max_zoom: int = 13,
+    matching_precision: int = 8,
+    fallback_to_higher_zoom: bool = False,
+    out_dtype: Optional[str] = "uint8",
+    out_nodata: NodataVal = None,
+    max_source_value: float = 10000.0,
+    max_output_value: Optional[float] = None,
+) -> ma.MaskedArray:
+    """
+    Scale input to different value range.
+
+    Inputs:
+    -------
+    inp
+        raster input to be scaled
+
+    Parameters:
+    -----------
+    bands : list
+        List of band indexes.
+    tresampling : str (default: 'nearest')
+        Resampling used when reading from mosaic.
+    matching_method : str ('gdal' or 'min') (default: 'gdal')
+        gdal: Uses GDAL's standard method. Here, the target resolution is
+            calculated by averaging the extent's pixel sizes over both x and y
+            axes. This approach returns a zoom level which may not have the
+            best quality but will speed up reading significantly.
+        min: Returns the zoom level which matches the minimum resolution of the
+            extents four corner pixels. This approach returns the zoom level
+            with the best possible quality but with low performance. If the
+            tile extent is outside of the destination pyramid, a
+            TopologicalError will be raised.
+    matching_max_zoom : int (optional, default: None)
+        If set, it will prevent reading from zoom levels above the maximum.
+    matching_precision : int (default: 8)
+        Round resolutions to n digits before comparing.
+    fallback_to_higher_zoom : bool (default: False)
+        In case no data is found at zoom level, try to read data from higher
+        zoom levels. Enabling this setting can lead to many IO requests in
+        areas with no data.
+    out_dtype: string
+        Output dtype for the target values, should fit the designated scaling from source,
+        if the output scaled values do not fit, they will be clipped to the output dtype.
+    out_nodata: float, int
+        Output Nodata, per default read from output nodata of the mapchete config.
+    max_source_value : float
+        Upper limit for clipping and scaling (e.g. 10000 for Sentinel-2).
+    max_output_value : float, None
+        Output value range (e.g. 255 for 8 bit). If None it will be determined by the out_dtype
+
+    Output:
+    -------
+    ma.ndarray
+        stretched input bands
+    """
+    logger.debug("read input mosaic")
+    if inp.is_empty():
+        logger.debug("mosaic empty")
+        raise MapcheteNodataTile
+    try:
+        mosaic = inp.read(
+            indexes=bands,
+            resampling=resampling,
+            matching_method=matching_method,
+            matching_max_zoom=matching_max_zoom,
+            matching_precision=matching_precision,
+            fallback_to_higher_zoom=fallback_to_higher_zoom,
+        ).astype(np.int16, copy=False)
+    except EmptyStackException:
+        logger.debug("mosaic empty: EmptyStackException")
+        raise MapcheteNodataTile
+    if mosaic[0].mask.all():
+        logger.debug("mosaic empty: all masked")
+        raise MapcheteNodataTile
+
+    if mp.output_params and mp.output_params.get("nodata") and out_nodata is None:
+        out_nodata = mp.output_params.get("nodata")
+    elif out_nodata is None:
+        logger.debug("Out nodata is None setting it to 0")
+        out_nodata = 0
+
+    logger.debug(
+        f"scale input raster values to wished dtype up to the max source value: {max_output_value}"
+    )
+    return dtype_scale(
+        bands=mosaic,
+        nodata=out_nodata,
+        out_dtype=out_dtype,
+        max_source_value=max_source_value,
+        max_output_value=max_output_value,
+    )

mapchete_eo/processes/eo_to_xarray.py

@@ -0,0 +1,19 @@
+from typing import List, Optional
+
+from rasterio.enums import Resampling
+from xarray import Dataset
+
+from mapchete_eo.base import EODataCube
+from mapchete_eo.types import MergeMethod
+
+
+def execute(
+    inp: EODataCube,
+    assets: Optional[List[str]] = None,
+    resampling: Resampling = Resampling.nearest,
+    merge_method: MergeMethod = MergeMethod.average,
+) -> Dataset:
+    """
+    Convert EO Data Cube into xarray.
+    """
+    return inp.read(assets=assets, resampling=resampling, merge_method=merge_method)

mapchete_eo/processes/merge_rasters.py

@@ -0,0 +1,239 @@
+import logging
+from enum import Enum
+from typing import List, Optional
+
+import numpy as np
+import numpy.ma as ma
+from mapchete import RasterInputGroup, VectorInputGroup, Timer
+from mapchete.errors import MapcheteNodataTile
+from mapchete.io.vector import to_shape
+from mapchete.processing.mp import MapcheteProcess
+from mapchete.tile import BufferedTile
+from rasterio.features import geometry_mask
+from shapely import unary_union
+from shapely.geometry import mapping, shape
+from shapely.geometry.base import BaseGeometry
+
+from mapchete_eo.image_operations import filters
+
+logger = logging.getLogger(__name__)
+
+
+class MergeMethod(str, Enum):
+    fill = "fill"
+    footprint_gradient = "footprint_gradient"
+
+
+def execute(
+    mp: MapcheteProcess,
+    rasters: RasterInputGroup,
+    vectors: VectorInputGroup,
+    gradient_buffer: int = 10,
+    merge_method: MergeMethod = MergeMethod.footprint_gradient,
+) -> ma.MaskedArray:
+    """
+    Merge multiple rasters into one.
+    """
+    raster_arrays = []
+    region_footprints = []
+
+    with Timer() as tt:
+        for raster_region, vector_region in zip(rasters, vectors):
+            # Vector Part
+            if vector_region is not None:
+                region_name_vector, region_vector = vector_region
+                region_geoms = region_vector.read()
+                if not region_geoms:
+                    logger.debug("%s vector is empty", region_name_vector)
+                    continue
+
+                # When there are multiple overlaps of aois/clipping creates multiple geoms,
+                # # make an union of all shapes, so that the rasters, vectors lists have the the same number of elements
+                region_geoms_shapes = []
+                for region_geom in region_geoms:
+                    region_geoms_shapes.append(shape(region_geom["geometry"]))
+
+                if len(region_geoms_shapes) > 1:
+                    region_geoms_shapes = unary_union(region_geoms_shapes)
+                    region_footprints.append(region_geoms_shapes)
+                else:
+                    region_footprints.append(shape(region_geoms[0]["geometry"]))
+
+            # Raster Part
+            region_name, region = raster_region
+
+            if region_name != region_name_vector:
+                raise ValueError(
+                    "Raster and Vector names should be the same to make sure they match itself, before area property of RasterInput works!"
+                )
+
+            raster = region.read()
+            if raster.mask.all():
+                logger.debug("%s raster is empty", region_name)
+                continue
+
+            raster_arrays.append(raster)
+
+            # This below wont work until area property of RasterInputs is working!
+            # if vector_region is None:
+            #     region_footprints.append(region.area)
+
+    logger.debug("%s rasters created in %s", len(raster_arrays), tt)
+
+    if len(raster_arrays) == 0:
+        raise MapcheteNodataTile("no input rasters found")
+
+    with Timer() as tt:
+        merged = merge_rasters(
+            raster_arrays,
+            mp.tile,
+            footprints=region_footprints,
+            method=merge_method,
+            gradient_buffer=gradient_buffer,
+        )
+    logger.debug("%s mosaics merged in %s", len(raster_arrays), tt)
+    return merged
+
+
+def merge_rasters(
+    rasters: List[ma.MaskedArray],
+    tile: BufferedTile,
+    method: MergeMethod = MergeMethod.fill,
+    footprints: Optional[List[BaseGeometry]] = None,
+    gradient_buffer: int = 10,
+) -> ma.MaskedArray:
+    footprints = footprints or []
+    if len(rasters) == 0:
+        raise ValueError("no rasters provided")
+    elif len(rasters) == 1:
+        return rasters[0]
+
+    if method == MergeMethod.fill:
+        return fillnodata_merge(rasters)
+
+    elif method == MergeMethod.footprint_gradient:
+        if footprints is None:
+            raise TypeError(
+                "for gradient_merge, a list of footprints has to be provided"
+            )
+        return gradient_merge(
+            rasters=rasters,
+            footprints=footprints,
+            tile=tile,
+            gradient_buffer=gradient_buffer,
+        )
+    else:  # pragma: no cover
+        raise ValueError(f"unkonw merge method '{method}'")
+
+
+def fillnodata_merge(
+    rasters: List[ma.MaskedArray],
+) -> ma.MaskedArray:
+    """
+    Read rasters sequentially and update masked pixels with values of next raster.
+    """
+    out = ma.empty_like(rasters[0])
+    for raster in rasters:
+        out[~raster.mask] = raster[~raster.mask]
+        out.mask[~raster.mask] = raster.mask[~raster.mask]
+        # if output is already full, don't add any further raster data
+        if not out.mask.any():
+            break
+    return out
+
+
+def gradient_merge(
+    rasters: List[ma.MaskedArray],
+    footprints: List[BaseGeometry],
+    tile: BufferedTile,
+    gradient_buffer: int = 10,
+) -> ma.MaskedArray:
+    """Use footprint geometries to merge rasters using a gradient buffer."""
+    if len(footprints) != len(rasters):  # pragma: no cover
+        raise ValueError(
+            f"footprints ({len(footprints)}) do not match rasters ({len(rasters)}) count"
+        )
+
+    out_data = np.zeros(rasters[0].shape, dtype=np.float16)
+    out_mask = np.ones(rasters[0].shape, dtype=bool)
+
+    for raster, footprint in zip(rasters, footprints):
+        # create gradient mask from footprint
+        footprint_geom = to_shape(footprint)
+        if footprint_geom.is_empty:
+            footprint_mask = np.ones(shape=raster.mask[0].shape, dtype=bool)
+        else:
+            footprint_mask = geometry_mask(
+                [mapping(footprint_geom)],
+                raster.mask[0].shape,
+                tile.transform,
+                all_touched=False,
+                invert=False,
+            )
+
+        # TODO: the gaussian_blur function demands a 3-band array, so we have to
+        # hack around that. This could be improved.
+        gradient_1band = filters.gaussian_blur(
+            (~np.stack([footprint_mask for _ in range(3)]) * 255).astype("uint8"),
+            radius=gradient_buffer,
+        )[0]
+        # gradient_1band now has values from 1 (no footprint coverage) to 255 (full
+        # footprint coverage)
+        # set 1 to 0:
+        gradient_1band[gradient_1band == 1] = 0
+        logger.debug(
+            f"gradient_1band; min: {np.min(gradient_1band)}, max: {np.max(gradient_1band)}"
+        )
+
+        # extrude array to match number of raster bands
+        gradient_8bit = np.stack([gradient_1band for _ in range(raster.shape[0])])
+        logger.debug(
+            f"gradient_8bit; min: {np.min(gradient_8bit)}, max: {np.max(gradient_8bit)}"
+        )
+
+        # scale gradient from 0 to 1
+        gradient = gradient_8bit / 255
+        logger.debug(f"gradient; min: {np.min(gradient)} , max: {np.max(gradient)}")
+
+        # now only apply the gradient where out and raster have values
+        # otherwise pick the remaining existing value or keep a masked
+        # pixel if both are masked
+
+        # clip raster with end of gradient:
+        clip_mask = raster.mask + (gradient_8bit == 0)
+        raster.mask = clip_mask
+
+        # the weight array is going to be used to merge the existing output array with
+        # current raster
+        weight = np.zeros(gradient.shape, dtype=np.float16)
+
+        # set weight values according to the following rules:
+        # both values available: use gradient (1 for full raster and 0 for full out)
+        weight[~out_mask & ~clip_mask] = gradient[~out_mask & ~clip_mask]
+        # only raster data available: 1
+        weight[out_mask & ~clip_mask] = 1.0
+        # only out data available: 0
+        weight[~out_mask & clip_mask] = 0.0
+        # none of them available: 0
+        weight[out_mask & clip_mask] = 0.0
+
+        # update out mask
+        weight_mask = np.zeros(weight.shape, dtype=bool)
+        # both values available: False
+        # only raster: False
+        # only out: False
+        # none: True
+        weight_mask[out_mask & clip_mask] = True
+
+        # sum of weighted existing data with new data
+        out_data[~clip_mask] = (
+            # weight existing data
+            (out_data[~clip_mask] * (1.0 - weight[~clip_mask]))
+            # weight new data
+            + (raster[~clip_mask].astype(np.float16) * weight[~clip_mask])
+        )
+        out_mask[~clip_mask] = weight_mask[~clip_mask]
+
+    return ma.MaskedArray(
+        data=out_data.astype(rasters[0].dtype, copy=False), mask=out_mask
+    )