mapchete-eo 2026.2.0__py3-none-any.whl

mapchete_eo/platforms/sentinel2/preconfigured_sources/guessers.py

@@ -0,0 +1,108 @@
+from typing import List
+
+from mapchete.path import MPathLike, MPath
+from pystac import Item
+
+from mapchete_eo.platforms.sentinel2.metadata_parser.base import S2MetadataPathMapper
+from mapchete_eo.platforms.sentinel2.metadata_parser.default_path_mapper import (
+    XMLMapper,
+)
+from mapchete_eo.platforms.sentinel2.metadata_parser.s2metadata import S2Metadata
+from mapchete_eo.platforms.sentinel2.preconfigured_sources.metadata_xml_mappers import (
+    EarthSearchPathMapper,
+    SinergisePathMapper,
+)
+
+
+def guess_metadata_path_mapper(
+    metadata_xml: MPathLike, **kwargs
+) -> S2MetadataPathMapper:
+    """Guess S2PathMapper based on URL.
+
+    If a new path mapper is added in this module, it should also be added to this function
+    in order to be detected.
+    """
+    metadata_xml = MPath.from_inp(metadata_xml)
+    if metadata_xml.startswith(
+        ("https://roda.sentinel-hub.com/sentinel-s2-l2a/", "s3://sentinel-s2-l2a/")
+    ) or metadata_xml.startswith(
+        ("https://roda.sentinel-hub.com/sentinel-s2-l1c/", "s3://sentinel-s2-l1c/")
+    ):
+        return SinergisePathMapper(metadata_xml, **kwargs)
+    elif metadata_xml.startswith(
+        "https://sentinel-cogs.s3.us-west-2.amazonaws.com/sentinel-s2-l2a-cogs/"
+    ):
+        return EarthSearchPathMapper(metadata_xml, **kwargs)
+    else:
+        return XMLMapper(metadata_xml, **kwargs)
+
+
+def guess_s2metadata_from_metadata_xml(metadata_xml: MPathLike, **kwargs) -> S2Metadata:
+    return S2Metadata.from_metadata_xml(
+        metadata_xml=metadata_xml,
+        path_mapper=guess_metadata_path_mapper(metadata_xml, **kwargs),
+        **kwargs,
+    )
+
+
+def guess_s2metadata_from_item(
+    item: Item,
+    metadata_assets: List[str] = ["metadata", "granule_metadata"],
+    boa_offset_fields: List[str] = [
+        "sentinel:boa_offset_applied",
+        "sentinel2:boa_offset_applied",
+        "earthsearch:boa_offset_applied",
+    ],
+    processing_baseline_fields: List[str] = [
+        "s2:processing_baseline",
+        "sentinel:processing_baseline",
+        "sentinel2:processing_baseline",
+        "processing:version",
+    ],
+    **kwargs,
+) -> S2Metadata:
+    """Custom code to initialize S2Metadata from a STAC item.
+
+    Depending on from which catalog the STAC item comes, this function should correctly
+    set all custom flags such as BOA offsets or pass on the correct path to the metadata XML
+    using the proper asset name.
+    """
+    metadata_assets = metadata_assets
+    for metadata_asset in metadata_assets:
+        if metadata_asset in item.assets:
+            metadata_path = MPath(item.assets[metadata_asset].href)
+            break
+    else:  # pragma: no cover
+        raise KeyError(
+            f"could not find path to metadata XML file in assets: {', '.join(item.assets.keys())}"
+        )
+
+    def _determine_offset():
+        for field in boa_offset_fields:
+            if item.properties.get(field):
+                return True
+
+        return False
+
+    boa_offset_applied = _determine_offset()
+
+    if metadata_path.is_remote() or metadata_path.is_absolute():
+        metadata_xml = metadata_path
+    else:
+        metadata_xml = MPath(item.self_href).parent / metadata_path
+    for processing_baseline_field in processing_baseline_fields:
+        try:
+            processing_baseline = item.properties[processing_baseline_field]
+            break
+        except KeyError:
+            pass
+    else:  # pragma: no cover
+        raise KeyError(
+            f"could not find processing baseline version in item properties: {item.properties}"
+        )
+    return guess_s2metadata_from_metadata_xml(
+        metadata_xml,
+        processing_baseline=processing_baseline,
+        boa_offset_applied=boa_offset_applied,
+        **kwargs,
+    )

mapchete_eo/platforms/sentinel2/preconfigured_sources/item_mappers.py

@@ -0,0 +1,171 @@
+from mapchete.path import MPath
+from pystac import Item
+
+from mapchete_eo.platforms.sentinel2._mapper_registry import (
+    maps_item_id,
+    maps_stac_metadata,
+    creates_s2metadata,
+)
+from mapchete_eo.platforms.sentinel2.preconfigured_sources.metadata_xml_mappers import (
+    CDSEPathMapper,
+    EarthSearchPathMapper,
+    EarthSearchC1PathMapper,
+    SinergisePathMapper,
+)
+from mapchete_eo.platforms.sentinel2.metadata_parser.s2metadata import S2Metadata
+from mapchete_eo.search.s2_mgrs import S2Tile
+
+
+# mapper functions decorated with metadata to have driver decide which one to apply when #
+##########################################################################################
+
+
+@maps_item_id(from_collections=["EarthSearch", "EarthSearch_legacy"])
+def earthsearch_id_mapper(item: Item) -> Item:
+    item.id = item.properties["s2:product_uri"].rstrip(".SAFE")
+    return item
+
+
+@maps_stac_metadata(from_collections=["EarthSearch"], to_data_archives=["AWSCOG"])
+def earthsearch_assets_paths_mapper(item: Item) -> Item:
+    """Nothing to do here as paths match catalog."""
+    return item
+
+
+@creates_s2metadata(from_collections=["EarthSearch"], to_metadata_archives=["roda"])
+def earthsearch_to_s2metadata(item: Item) -> S2Metadata:
+    return S2Metadata.from_stac_item(
+        item,
+        path_mapper=EarthSearchC1PathMapper(
+            MPath(item.assets["granule_metadata"].href)
+        ),
+        processing_baseline_field="s2:processing_baseline",
+    )
+
+
+@creates_s2metadata(
+    from_collections=["EarthSearch_legacy"], to_metadata_archives=["roda"]
+)
+def earthsearch_legacy_to_s2metadata(item: Item) -> S2Metadata:
+    return S2Metadata.from_stac_item(
+        item,
+        path_mapper=EarthSearchPathMapper(MPath(item.assets["granule_metadata"].href)),
+        boa_offset_field="earthsearch:boa_offset_applied",
+        processing_baseline_field="s2:processing_baseline",
+    )
+
+
+@maps_item_id(from_collections=["CDSE"])
+def plain_id_mapper(item: Item) -> Item:
+    return item
+
+
+CDSE_ASSET_NAME_MAPPING = {
+    "AOT_10m": "aot",
+    "B01_20m": "coastal",
+    "B02_10m": "blue",
+    "B03_10m": "green",
+    "B04_10m": "red",
+    "B05_20m": "rededge1",
+    "B06_20m": "rededge2",
+    "B07_20m": "rededge3",
+    "B08_10m": "nir",
+    "B09_60m": "nir09",
+    "B11_20m": "swir16",
+    "B12_20m": "swir22",
+    "B8A_20m": "nir08",
+    "SCL_20m": "scl",
+    "TCI_10m": "visual",
+    "WVP_10m": "wvp",
+}
+
+
+@maps_stac_metadata(from_collections=["CDSE"])
+def cdse_asset_names(item: Item) -> Item:
+    new_assets = {}
+    for asset_name, asset in item.assets.items():
+        if asset_name in CDSE_ASSET_NAME_MAPPING:
+            asset_name = CDSE_ASSET_NAME_MAPPING[asset_name]
+        new_assets[asset_name] = asset
+
+    item.assets = new_assets
+
+    item.properties["s2:datastrip_id"] = item.properties.get("eopf:datastrip_id")
+    return item
+
+
+@maps_stac_metadata(from_collections=["CDSE"], to_data_archives=["AWSJP2"])
+def map_cdse_paths_to_jp2_archive(item: Item) -> Item:
+    """
+    CSDE has the following assets:
+    AOT_10m, AOT_20m, AOT_60m, B01_20m, B01_60m, B02_10m, B02_20m, B02_60m, B03_10m, B03_20m,
+    B03_60m, B04_10m, B04_20m, B04_60m, B05_20m, B05_60m, B06_20m, B06_60m, B07_20m, B07_60m,
+    B08_10m, B09_60m, B11_20m, B11_60m, B12_20m, B12_60m, B8A_20m, B8A_60m, Product, SCL_20m,
+    SCL_60m, TCI_10m, TCI_20m, TCI_60m, WVP_10m, WVP_20m, WVP_60m, thumbnail, safe_manifest,
+    granule_metadata, inspire_metadata, product_metadata, datastrip_metadata
+
+    sample path for AWS JP2:
+    s3://sentinel-s2-l2a/tiles/51/K/XR/2020/7/31/0/R10m/
+    """
+    if item.datetime is None:
+        raise ValueError(f"product {item.get_self_href()} does not have a timestamp")
+    path_base_scheme = "s3://sentinel-s2-l2a/tiles/{utm_zone}/{latitude_band}/{grid_square}/{year}/{month}/{day}/{count}"
+    s2tile = S2Tile.from_grid_code(item.properties["grid:code"])
+    product_basepath = MPath(
+        path_base_scheme.format(
+            utm_zone=int(s2tile.utm_zone),
+            latitude_band=s2tile.latitude_band,
+            grid_square=s2tile.grid_square,
+            year=item.datetime.year,
+            month=item.datetime.month,
+            day=item.datetime.day,
+            count=0,  # TODO: get count dynamically from metadata
+        )
+    )
+    new_assets = {}
+    for asset_name, asset in item.assets.items():
+        # ignore these assets
+        if asset_name in [
+            "Product",
+            "safe_manifest",
+            "product_metadata",
+            "inspire_metadata",
+            "datastrip_metadata",
+        ]:
+            continue
+        # set thumbnnail
+        elif asset_name == "thumbnail":
+            asset.href = str(product_basepath / "R60m" / "TCI.jp2")
+        # point to proper metadata
+        elif asset_name == "granule_metadata":
+            asset.href = str(product_basepath / "metadata.xml")
+        # change band asset names and point to their new locations
+        elif asset_name in CDSE_ASSET_NAME_MAPPING:
+            name, resolution = asset_name.split("_")
+            asset.href = product_basepath / f"R{resolution}" / f"{name}.jp2"
+            asset_name = CDSE_ASSET_NAME_MAPPING[asset_name]
+        else:
+            continue
+        new_assets[asset_name] = asset
+
+    item.assets = new_assets
+
+    return item
+
+
+@creates_s2metadata(from_collections=["CDSE"], to_metadata_archives=["CDSE"])
+def cdse_s2metadata(item: Item) -> S2Metadata:
+    return S2Metadata.from_stac_item(
+        item,
+        path_mapper=CDSEPathMapper(MPath(item.assets["granule_metadata"].href)),
+        processing_baseline_field="processing:version",
+    )
+
+
+@creates_s2metadata(from_collections=["CDSE"], to_metadata_archives=["roda"])
+def cdse_to_roda_s2metadata(item: Item) -> S2Metadata:
+    return S2Metadata.from_stac_item(
+        item,
+        path_mapper=SinergisePathMapper(MPath(item.assets["granule_metadata"].href)),
+        processing_baseline_field="processing:version",
+    )

mapchete_eo/platforms/sentinel2/preconfigured_sources/metadata_xml_mappers.py

@@ -0,0 +1,217 @@
+from mapchete.path import MPath, MPathLike
+
+from mapchete_eo.platforms.sentinel2.metadata_parser.base import S2MetadataPathMapper
+from mapchete_eo.platforms.sentinel2.processing_baseline import ProcessingBaseline
+from mapchete_eo.platforms.sentinel2.types import (
+    BandQI,
+    L2ABand,
+    ProductQI,
+    ProductQIMaskResolution,
+)
+
+
+class SinergisePathMapper(S2MetadataPathMapper):
+    """
+    Return true paths of product quality assets from the Sinergise S2 bucket.
+
+    e.g.:
+    B01 detector footprints: s3://sentinel-s2-l2a/tiles/51/K/XR/2020/7/31/0/qi/MSK_DETFOO_B01.gml
+    Cloud masks: s3://sentinel-s2-l2a/tiles/51/K/XR/2020/7/31/0/qi/MSK_CLOUDS_B00.gml
+
+    newer products however:
+    B01 detector footprints: s3://sentinel-s2-l2a/tiles/51/K/XR/2022/6/6/0/qi/DETFOO_B01.jp2
+    no vector cloudmasks available anymore
+    """
+
+    _PRE_0400_MASK_PATHS = {
+        ProductQI.classification: "MSK_CLOUDS_B00.gml",
+        ProductQI.cloud_probability: "CLD_{resolution}.jp2",  # are they really there?
+        ProductQI.snow_probability: "SNW_{resolution}.jp2",  # are they really there?
+        BandQI.detector_footprints: "MSK_DETFOO_{band_identifier}.gml",
+        BandQI.technical_quality: "MSK_TECQUA_{band_identifier}.gml",
+    }
+    _POST_0400_MASK_PATHS = {
+        ProductQI.classification: "CLASSI_B00.jp2",
+        ProductQI.cloud_probability: "CLD_{resolution}.jp2",
+        ProductQI.snow_probability: "SNW_{resolution}.jp2",
+        BandQI.detector_footprints: "DETFOO_{band_identifier}.jp2",
+        BandQI.technical_quality: "QUALIT_{band_identifier}.jp2",
+    }
+
+    def __init__(
+        self,
+        url: MPathLike,
+        bucket: str = "sentinel-s2-l2a",
+        protocol: str = "s3",
+        baseline_version: str = "04.00",
+        **kwargs,
+    ):
+        url = MPath.from_inp(url)
+        tileinfo_path = url.parent / "tileInfo.json"
+        self._path = MPath(
+            "/".join(tileinfo_path.elements[-9:-1]), **tileinfo_path._kwargs
+        )
+        self._utm_zone, self._latitude_band, self._grid_square = self._path.split("/")[
+            1:-4
+        ]
+        self._baseurl = bucket
+        self._protocol = protocol
+        self.processing_baseline = ProcessingBaseline.from_version(baseline_version)
+
+    def product_qi_mask(
+        self,
+        qi_mask: ProductQI,
+        resolution: ProductQIMaskResolution = ProductQIMaskResolution["60m"],
+    ) -> MPath:
+        """Determine product QI mask according to Sinergise bucket schema."""
+        if self.processing_baseline.version < "04.00":
+            mask_path = self._PRE_0400_MASK_PATHS[qi_mask]
+        else:
+            mask_path = self._POST_0400_MASK_PATHS[qi_mask]
+        key = f"{self._path}/qi/{mask_path.format(resolution=resolution.name)}"
+        return MPath.from_inp(f"{self._protocol}://{self._baseurl}/{key}")
+
+    def classification_mask(self) -> MPath:
+        return self.product_qi_mask(ProductQI.classification)
+
+    def cloud_probability_mask(
+        self, resolution: ProductQIMaskResolution = ProductQIMaskResolution["60m"]
+    ) -> MPath:
+        return self.product_qi_mask(ProductQI.cloud_probability, resolution=resolution)
+
+    def snow_probability_mask(
+        self, resolution: ProductQIMaskResolution = ProductQIMaskResolution["60m"]
+    ) -> MPath:
+        return self.product_qi_mask(ProductQI.snow_probability, resolution=resolution)
+
+    def band_qi_mask(self, qi_mask: BandQI, band: L2ABand) -> MPath:
+        """Determine product QI mask according to Sinergise bucket schema."""
+        try:
+            if self.processing_baseline.version < "04.00":
+                mask_path = self._PRE_0400_MASK_PATHS[qi_mask]
+            else:
+                mask_path = self._POST_0400_MASK_PATHS[qi_mask]
+        except KeyError:
+            raise DeprecationWarning(
+                f"'{qi_mask.name}' quality mask not found in this product"
+            )
+        key = f"{self._path}/qi/{mask_path.format(band_identifier=band.name)}"
+        return MPath.from_inp(f"{self._protocol}://{self._baseurl}/{key}")
+
+    def technical_quality_mask(self, band: L2ABand) -> MPath:
+        return self.band_qi_mask(BandQI.technical_quality, band)
+
+    def detector_footprints(self, band: L2ABand) -> MPath:
+        return self.band_qi_mask(BandQI.detector_footprints, band)
+
+
+class EarthSearchPathMapper(SinergisePathMapper):
+    """
+    The COG archive maintained by E84 and covered by EarthSearch does not hold additional data
+    such as the GML files. This class maps the metadata masks to the current EarthSearch product.
+
+    e.g.:
+    B01 detector footprints: s3://sentinel-s2-l2a/tiles/51/K/XR/2020/7/31/0/qi/MSK_DETFOO_B01.gml
+    Cloud masks: s3://sentinel-s2-l2a/tiles/51/K/XR/2020/7/31/0/qi/MSK_CLOUDS_B00.gml
+
+    newer products however:
+    B01 detector footprints: s3://sentinel-s2-l2a/tiles/51/K/XR/2022/6/6/0/qi/DETFOO_B01.jp2
+    no vector cloudmasks available anymore
+    """
+
+    def __init__(
+        self,
+        metadata_xml: MPath,
+        alternative_metadata_baseurl: str = "sentinel-s2-l2a",
+        protocol: str = "s3",
+        baseline_version: str = "04.00",
+        **kwargs,
+    ):
+        basedir = metadata_xml.parent
+        self._path = (basedir / "tileinfo_metadata.json").read_json()["path"]
+        self._utm_zone, self._latitude_band, self._grid_square = basedir.elements[-6:-3]
+        self._baseurl = alternative_metadata_baseurl
+        self._protocol = protocol
+        self.processing_baseline = ProcessingBaseline.from_version(baseline_version)
+
+
+class EarthSearchC1PathMapper(SinergisePathMapper):
+    """
+    The newer C1 collection has cloud and snow probability masks as assets, so we only need to
+    map to the rest.
+    """
+
+    def __init__(
+        self,
+        metadata_xml: MPath,
+        alternative_metadata_baseurl: str = "sentinel-s2-l2a",
+        protocol: str = "s3",
+        baseline_version: str = "04.00",
+        **kwargs,
+    ):
+        basedir = metadata_xml.parent
+        self._path = (basedir / "tileInfo.json").read_json()["path"]
+        self._utm_zone, self._latitude_band, self._grid_square = basedir.elements[-6:-3]
+        self._baseurl = alternative_metadata_baseurl
+        self._protocol = protocol
+        self.processing_baseline = ProcessingBaseline.from_version(baseline_version)
+
+
+class CDSEPathMapper(S2MetadataPathMapper):
+    _MASK_FILENAMES = {
+        ProductQI.classification: "MSK_CLASSI_B00.jp2",
+        ProductQI.cloud_probability: "MSK_CLDPRB_{resolution}.jp2",
+        ProductQI.snow_probability: "MSK_SNWPRB_{resolution}.jp2",
+        BandQI.detector_footprints: "MSK_DETFOO_{band_identifier}.jp2",
+        BandQI.technical_quality: "MSK_QUALIT_{band_identifier}.jp2",
+    }
+
+    def __init__(
+        self,
+        url: MPathLike,
+        baseline_version: str = "04.00",
+        **kwargs,
+    ):
+        url = MPath.from_inp(url)
+        self._path = url.parent
+        self.processing_baseline = ProcessingBaseline.from_version(baseline_version)
+
+    def product_qi_mask(
+        self,
+        qi_mask: ProductQI,
+        resolution: ProductQIMaskResolution = ProductQIMaskResolution["60m"],
+    ) -> MPath:
+        """Determine product QI mask according to Sinergise bucket schema."""
+        mask_path = self._MASK_FILENAMES[qi_mask]
+        key = f"QI_DATA/{mask_path.format(resolution=resolution.name)}"
+        return self._path / key
+
+    def classification_mask(self) -> MPath:
+        return self.product_qi_mask(ProductQI.classification)
+
+    def cloud_probability_mask(
+        self, resolution: ProductQIMaskResolution = ProductQIMaskResolution["60m"]
+    ) -> MPath:
+        return self.product_qi_mask(ProductQI.cloud_probability, resolution=resolution)
+
+    def snow_probability_mask(
+        self, resolution: ProductQIMaskResolution = ProductQIMaskResolution["60m"]
+    ) -> MPath:
+        return self.product_qi_mask(ProductQI.snow_probability, resolution=resolution)
+
+    def band_qi_mask(self, qi_mask: BandQI, band: L2ABand) -> MPath:
+        """Determine product QI mask according to Sinergise bucket schema."""
+        try:
+            mask_path = self._MASK_FILENAMES[qi_mask]
+        except KeyError:
+            raise DeprecationWarning(
+                f"'{qi_mask.name}' quality mask not found in this product"
+            )
+        key = f"QI_DATA/{mask_path.format(band_identifier=band.name)}"
+        return self._path / key
+
+    def technical_quality_mask(self, band: L2ABand) -> MPath:
+        return self.band_qi_mask(BandQI.technical_quality, band)
+
+    def detector_footprints(self, band: L2ABand) -> MPath:
+        return self.band_qi_mask(BandQI.detector_footprints, band)

mapchete_eo/platforms/sentinel2/preprocessing_tasks.py

@@ -0,0 +1,50 @@
+import logging
+from typing import Optional, Union
+
+import pystac
+
+from mapchete_eo.exceptions import CorruptedProductMetadata
+from mapchete_eo.io.items import get_item_property
+from mapchete_eo.platforms.sentinel2.config import CacheConfig
+from mapchete_eo.platforms.sentinel2.product import S2Product
+from mapchete_eo.platforms.sentinel2.source import Sentinel2Source
+from mapchete_eo.product import add_to_blacklist
+from mapchete_eo.settings import mapchete_eo_settings
+
+logger = logging.getLogger(__name__)
+
+
+def parse_s2_product(
+    item: pystac.Item,
+    cache_config: Optional[CacheConfig] = None,
+    cache_all: bool = False,
+) -> Union[S2Product, CorruptedProductMetadata]:
+    """
+    Parse a Sentinel-2 STAC Item into an S2Product.
+    """
+    # use mapper from source if applicable
+    source: Union[Sentinel2Source, None] = item.properties.pop(
+        "mapchete_eo:source", None
+    )
+    try:
+        s2product = S2Product.from_stac_item(
+            item,
+            cache_config=cache_config,
+            cache_all=cache_all,
+            metadata_mapper=None if source is None else source.get_s2metadata_mapper(),
+            item_modifier_funcs=None if source is None else source.item_modifier_funcs,
+            lazy_load_item=mapchete_eo_settings.lazy_load_stac_items,
+            item_property_cache={
+                key: get_item_property(item, key)
+                for key in [
+                    "datetime",
+                    "eo:cloud_cover",
+                    "id",
+                    "s2:datastrip_id",
+                ]
+            },
+        )
+    except CorruptedProductMetadata as exc:
+        add_to_blacklist(item.get_self_href())
+        return exc
+    return s2product