mapchete-eo 2025.10.0__py2.py3-none-any.whl → 2025.11.0__py2.py3-none-any.whl

mapchete_eo/platforms/sentinel2/{metadata_parser.py → metadata_parser/s2metadata.py}

@@ -6,14 +6,12 @@ sun angles, quality masks, etc.
 from __future__ import annotations
 
 import logging
-import warnings
 from functools import cached_property
-from typing import Any, Callable, Dict, List, Optional, Union
+from typing import Any, Dict, List, Optional, Tuple, Union
 from xml.etree.ElementTree import Element, ParseError
 
 import numpy as np
 import numpy.ma as ma
-from pydantic import BaseModel
 import pystac
 from affine import Affine
 from fiona.transform import transform_geom
@@ -33,9 +31,17 @@ from tilematrix import Shape
 
 from mapchete_eo.exceptions import AssetEmpty, AssetMissing, CorruptedProductMetadata
 from mapchete_eo.io import open_xml, read_mask_as_raster
-from mapchete_eo.platforms.sentinel2.path_mappers import default_path_mapper_guesser
-from mapchete_eo.platforms.sentinel2.path_mappers.base import S2PathMapper
-from mapchete_eo.platforms.sentinel2.path_mappers.metadata_xml import XMLMapper
+from mapchete_eo.io.items import get_item_property
+from mapchete_eo.io.path import asset_mpath
+from mapchete_eo.platforms.sentinel2.metadata_parser.models import (
+    ViewingIncidenceAngles,
+    SunAngleData,
+    SunAnglesData,
+)
+from mapchete_eo.platforms.sentinel2.metadata_parser.base import S2MetadataPathMapper
+from mapchete_eo.platforms.sentinel2.metadata_parser.default_path_mapper import (
+    XMLMapper,
+)
 from mapchete_eo.platforms.sentinel2.processing_baseline import ProcessingBaseline
 from mapchete_eo.platforms.sentinel2.types import (
     BandQI,
@@ -59,77 +65,12 @@ def open_granule_metadata_xml(metadata_xml: MPath) -> Element:
         raise CorruptedProductMetadata(exc)
 
 
-def s2metadata_from_stac_item(
-    item: pystac.Item,
-    metadata_assets: List[str] = ["metadata", "granule_metadata"],
-    boa_offset_fields: List[str] = [
-        "sentinel:boa_offset_applied",
-        "sentinel2:boa_offset_applied",
-        "earthsearch:boa_offset_applied",
-    ],
-    processing_baseline_fields: List[str] = [
-        "s2:processing_baseline",
-        "sentinel:processing_baseline",
-        "sentinel2:processing_baseline",
-        "processing:version",
-    ],
-    **kwargs,
-) -> S2Metadata:
-    """Custom code to initialize S2Metadata from a STAC item.
-
-    Depending on from which catalog the STAC item comes, this function should correctly
-    set all custom flags such as BOA offsets or pass on the correct path to the metadata XML
-    using the proper asset name.
-    """
-    metadata_assets = metadata_assets
-    for metadata_asset in metadata_assets:
-        if metadata_asset in item.assets:
-            metadata_path = MPath(item.assets[metadata_asset].href)
-            break
-    else:  # pragma: no cover
-        raise KeyError(
-            f"could not find path to metadata XML file in assets: {', '.join(item.assets.keys())}"
-        )
-
-    def _determine_offset():
-        for field in boa_offset_fields:
-            if item.properties.get(field):
-                return True
-
-        return False
-
-    boa_offset_applied = _determine_offset()
-
-    if metadata_path.is_remote() or metadata_path.is_absolute():
-        metadata_xml = metadata_path
-    else:
-        metadata_xml = MPath(item.self_href).parent / metadata_path
-    for processing_baseline_field in processing_baseline_fields:
-        try:
-            processing_baseline = item.properties[processing_baseline_field]
-            break
-        except KeyError:
-            pass
-    else:  # pragma: no cover
-        raise KeyError(
-            f"could not find processing baseline version in item properties: {item.properties}"
-        )
-    return S2Metadata.from_metadata_xml(
-        metadata_xml=metadata_xml,
-        processing_baseline=processing_baseline,
-        boa_offset_applied=boa_offset_applied,
-        **kwargs,
-    )
-
-
 class S2Metadata:
     metadata_xml: MPath
-    path_mapper: S2PathMapper
+    path_mapper: S2MetadataPathMapper
     processing_baseline: ProcessingBaseline
     boa_offset_applied: bool = False
     _cached_xml_root: Optional[Element] = None
-    path_mapper_guesser: Callable = default_path_mapper_guesser
-    from_stac_item_constructor: Callable = s2metadata_from_stac_item
     crs: CRS
     bounds: Bounds
     footprint: Union[Polygon, MultiPolygon]
@@ -138,7 +79,7 @@ class S2Metadata:
     def __init__(
         self,
         metadata_xml: MPath,
-        path_mapper: S2PathMapper,
+        path_mapper: S2MetadataPathMapper,
         xml_root: Optional[Element] = None,
         boa_offset_applied: bool = False,
         **kwargs,
@@ -161,10 +102,8 @@ class S2Metadata:
         return f"<S2Metadata id={self.product_id}, processing_baseline={self.processing_baseline}>"
 
     def clear_cached_data(self):
-        logger.debug("clear S2Metadata internal caches")
         self._cache = dict(viewing_incidence_angles=dict(), detector_footprints=dict())
         if self._cached_xml_root is not None:
-            logger.debug("clear S2Metadata xml cache")
             self._cached_xml_root.clear()
             self._cached_xml_root = None
         self.path_mapper.clear_cached_data()
@@ -188,19 +127,15 @@ class S2Metadata:
     def from_metadata_xml(
         cls,
         metadata_xml: Union[str, MPath],
+        path_mapper: Optional[S2MetadataPathMapper] = None,
         processing_baseline: Optional[str] = None,
-        path_mapper: Optional[S2PathMapper] = None,
         **kwargs,
     ) -> S2Metadata:
         metadata_xml = MPath.from_inp(metadata_xml, **kwargs)
         xml_root = open_granule_metadata_xml(metadata_xml)
+
         if path_mapper is None:
-            # guess correct path mapper
-            path_mapper = cls.path_mapper_guesser(
-                metadata_xml,
-                xml_root=xml_root,
-                **kwargs,
-            )
+            path_mapper = XMLMapper(metadata_xml=metadata_xml, xml_root=xml_root)
 
         # use processing baseline version from argument if available
         if processing_baseline:
@@ -219,9 +154,38 @@ class S2Metadata:
             metadata_xml, path_mapper=path_mapper, xml_root=xml_root, **kwargs
         )
 
-    @classmethod
-    def from_stac_item(cls, item: pystac.Item, **kwargs) -> S2Metadata:
-        return cls.from_stac_item_constructor(item, **kwargs)
+    @staticmethod
+    def from_stac_item(
+        item: pystac.Item,
+        metadata_xml_asset_name: Tuple[str, ...] = ("metadata", "granule_metadata"),
+        boa_offset_field: Union[str, Tuple[str, ...]] = (
+            "earthsearch:boa_offset_applied"
+        ),
+        processing_baseline_field: Union[str, Tuple[str, ...]] = (
+            "s2:processing_baseline",
+            "sentinel2:processing_baseline",
+            "processing:version",
+        ),
+        **kwargs,
+    ) -> S2Metadata:
+        # try to find path to metadata.xml
+        metadata_xml_path = asset_mpath(item, metadata_xml_asset_name)
+        # make path absolute
+        if not (metadata_xml_path.is_remote() or metadata_xml_path.is_absolute()):
+            metadata_xml_path = MPath(item.self_href).parent / metadata_xml_path
+
+        # try to find information on processing baseline version
+        processing_baseline = get_item_property(item, processing_baseline_field)
+
+        # see if boa_offset_applied flag is available
+        boa_offset_applied = get_item_property(item, boa_offset_field, default=False)
+
+        return S2Metadata.from_metadata_xml(
+            metadata_xml=metadata_xml_path,
+            processing_baseline=processing_baseline,
+            boa_offset_applied=boa_offset_applied,
+            **kwargs,
+        )
 
     @property
     def xml_root(self):
@@ -272,13 +236,13 @@ class S2Metadata:
         for product_qi_mask in ProductQI:
             if product_qi_mask == ProductQI.classification:
                 out[product_qi_mask.name] = self.path_mapper.product_qi_mask(
-                    product_qi_mask
+                    qi_mask=product_qi_mask
                 )
             else:
                 for resolution in ProductQIMaskResolution:
                     out[f"{product_qi_mask.name}-{resolution.name}"] = (
                         self.path_mapper.product_qi_mask(
-                            product_qi_mask, resolution=resolution
+                            qi_mask=product_qi_mask, resolution=resolution
                         )
                     )
 
@@ -589,65 +553,6 @@ class S2Metadata:
         return mean
 
 
-class SunAngleData(BaseModel):
-    model_config = dict(arbitrary_types_allowed=True)
-    raster: ReferencedRaster
-    mean: float
-
-
-class SunAnglesData(BaseModel):
-    azimuth: SunAngleData
-    zenith: SunAngleData
-
-    def get_angle(self, angle: SunAngle) -> SunAngleData:
-        if angle == SunAngle.azimuth:
-            return self.azimuth
-        elif angle == SunAngle.zenith:
-            return self.zenith
-        else:
-            raise KeyError(f"unknown angle: {angle}")
-
-
-class ViewingIncidenceAngle(BaseModel):
-    model_config = dict(arbitrary_types_allowed=True)
-    detectors: Dict[int, ReferencedRaster]
-    mean: float
-
-    def merge_detectors(
-        self, fill_edges: bool = True, smoothing_iterations: int = 3
-    ) -> ReferencedRaster:
-        if not self.detectors:
-            raise CorruptedProductMetadata("no viewing incidence angles available")
-        sample = next(iter(self.detectors.values()))
-        with warnings.catch_warnings():
-            warnings.simplefilter("ignore", category=RuntimeWarning)
-            merged = np.nanmean(
-                np.stack([raster.data for raster in self.detectors.values()]), axis=0
-            )
-        if fill_edges:
-            merged = fillnodata(
-                ma.masked_invalid(merged), smoothing_iterations=smoothing_iterations
-            )
-        return ReferencedRaster.from_array_like(
-            array_like=ma.masked_invalid(merged),
-            transform=sample.transform,
-            crs=sample.crs,
-        )
-
-
-class ViewingIncidenceAngles(BaseModel):
-    azimuth: ViewingIncidenceAngle
-    zenith: ViewingIncidenceAngle
-
-    def get_angle(self, angle: ViewAngle) -> ViewingIncidenceAngle:
-        if angle == ViewAngle.azimuth:
-            return self.azimuth
-        elif angle == ViewAngle.zenith:
-            return self.zenith
-        else:
-            raise KeyError(f"unknown angle: {angle}")
-
-
 def _get_grids(root: Element, crs: CRS) -> Dict[Resolution, Grid]:
     geoinfo = {
         Resolution["10m"]: dict(crs=crs),

mapchete_eo/platforms/sentinel2/preconfigured_sources/__init__.py

@@ -0,0 +1,57 @@
+from typing import Dict, Any
+
+# importing this is crucial so the mapping functions get registered before registry is accessed
+from mapchete_eo.platforms.sentinel2.preconfigured_sources.item_mappers import (
+    earthsearch_assets_paths_mapper,
+    earthsearch_id_mapper,
+    earthsearch_to_s2metadata,
+    cdse_asset_names,
+    cdse_s2metadata,
+)
+from mapchete_eo.platforms.sentinel2.preconfigured_sources.guessers import (
+    guess_metadata_path_mapper,
+    guess_s2metadata_from_item,
+    guess_s2metadata_from_metadata_xml,
+)
+
+
+__all__ = [
+    "guess_metadata_path_mapper",
+    "guess_s2metadata_from_item",
+    "guess_s2metadata_from_metadata_xml",
+    "earthsearch_assets_paths_mapper",
+    "earthsearch_id_mapper",
+    "earthsearch_to_s2metadata",
+    "cdse_asset_names",
+    "cdse_s2metadata",
+]
+
+KNOWN_SOURCES: Dict[str, Any] = {
+    "EarthSearch": {
+        "collection": "https://earth-search.aws.element84.com/v1/collections/sentinel-2-c1-l2a",
+    },
+    "EarthSearch_legacy": {
+        "collection": "https://earth-search.aws.element84.com/v1/collections/sentinel-2-l2a",
+    },
+    "CDSE": {
+        "collection": "https://stac.dataspace.copernicus.eu/v1/collections/sentinel-2-l2a",
+        "metadata_archive": "CDSE",
+    },
+}
+
+DEPRECATED_ARCHIVES = {
+    "S2AWS_COG": {
+        "collection": "https://earth-search.aws.element84.com/v1/collections/sentinel-2-c1-l2a",
+    },
+    "S2AWS_JP2": {
+        "collection": "https://stac.dataspace.copernicus.eu/v1/collections/sentinel-2-l2a",
+        "data_archive": "AWSJP2",
+    },
+    "S2CDSE_AWSJP2": {
+        "collection": "https://stac.dataspace.copernicus.eu/v1/collections/sentinel-2-l2a",
+        "data_archive": "AWSJP2",
+    },
+    "S2CDSE_JP2": {
+        "collection": "https://stac.dataspace.copernicus.eu/v1/collections/sentinel-2-l2a",
+    },
+}

mapchete_eo/platforms/sentinel2/preconfigured_sources/guessers.py

@@ -0,0 +1,108 @@
+from typing import List
+
+from mapchete.path import MPathLike, MPath
+from pystac import Item
+
+from mapchete_eo.platforms.sentinel2.metadata_parser.base import S2MetadataPathMapper
+from mapchete_eo.platforms.sentinel2.metadata_parser.default_path_mapper import (
+    XMLMapper,
+)
+from mapchete_eo.platforms.sentinel2.metadata_parser.s2metadata import S2Metadata
+from mapchete_eo.platforms.sentinel2.preconfigured_sources.metadata_xml_mappers import (
+    EarthSearchPathMapper,
+    SinergisePathMapper,
+)
+
+
+def guess_metadata_path_mapper(
+    metadata_xml: MPathLike, **kwargs
+) -> S2MetadataPathMapper:
+    """Guess S2PathMapper based on URL.
+
+    If a new path mapper is added in this module, it should also be added to this function
+    in order to be detected.
+    """
+    metadata_xml = MPath.from_inp(metadata_xml)
+    if metadata_xml.startswith(
+        ("https://roda.sentinel-hub.com/sentinel-s2-l2a/", "s3://sentinel-s2-l2a/")
+    ) or metadata_xml.startswith(
+        ("https://roda.sentinel-hub.com/sentinel-s2-l1c/", "s3://sentinel-s2-l1c/")
+    ):
+        return SinergisePathMapper(metadata_xml, **kwargs)
+    elif metadata_xml.startswith(
+        "https://sentinel-cogs.s3.us-west-2.amazonaws.com/sentinel-s2-l2a-cogs/"
+    ):
+        return EarthSearchPathMapper(metadata_xml, **kwargs)
+    else:
+        return XMLMapper(metadata_xml, **kwargs)
+
+
+def guess_s2metadata_from_metadata_xml(metadata_xml: MPathLike, **kwargs) -> S2Metadata:
+    return S2Metadata.from_metadata_xml(
+        metadata_xml=metadata_xml,
+        path_mapper=guess_metadata_path_mapper(metadata_xml, **kwargs),
+        **kwargs,
+    )
+
+
+def guess_s2metadata_from_item(
+    item: Item,
+    metadata_assets: List[str] = ["metadata", "granule_metadata"],
+    boa_offset_fields: List[str] = [
+        "sentinel:boa_offset_applied",
+        "sentinel2:boa_offset_applied",
+        "earthsearch:boa_offset_applied",
+    ],
+    processing_baseline_fields: List[str] = [
+        "s2:processing_baseline",
+        "sentinel:processing_baseline",
+        "sentinel2:processing_baseline",
+        "processing:version",
+    ],
+    **kwargs,
+) -> S2Metadata:
+    """Custom code to initialize S2Metadata from a STAC item.
+
+    Depending on from which catalog the STAC item comes, this function should correctly
+    set all custom flags such as BOA offsets or pass on the correct path to the metadata XML
+    using the proper asset name.
+    """
+    metadata_assets = metadata_assets
+    for metadata_asset in metadata_assets:
+        if metadata_asset in item.assets:
+            metadata_path = MPath(item.assets[metadata_asset].href)
+            break
+    else:  # pragma: no cover
+        raise KeyError(
+            f"could not find path to metadata XML file in assets: {', '.join(item.assets.keys())}"
+        )
+
+    def _determine_offset():
+        for field in boa_offset_fields:
+            if item.properties.get(field):
+                return True
+
+        return False
+
+    boa_offset_applied = _determine_offset()
+
+    if metadata_path.is_remote() or metadata_path.is_absolute():
+        metadata_xml = metadata_path
+    else:
+        metadata_xml = MPath(item.self_href).parent / metadata_path
+    for processing_baseline_field in processing_baseline_fields:
+        try:
+            processing_baseline = item.properties[processing_baseline_field]
+            break
+        except KeyError:
+            pass
+    else:  # pragma: no cover
+        raise KeyError(
+            f"could not find processing baseline version in item properties: {item.properties}"
+        )
+    return guess_s2metadata_from_metadata_xml(
+        metadata_xml,
+        processing_baseline=processing_baseline,
+        boa_offset_applied=boa_offset_applied,
+        **kwargs,
+    )

mapchete_eo/platforms/sentinel2/preconfigured_sources/item_mappers.py

@@ -0,0 +1,171 @@
+from mapchete.path import MPath
+from pystac import Item
+
+from mapchete_eo.platforms.sentinel2._mapper_registry import (
+    maps_item_id,
+    maps_stac_metadata,
+    creates_s2metadata,
+)
+from mapchete_eo.platforms.sentinel2.preconfigured_sources.metadata_xml_mappers import (
+    CDSEPathMapper,
+    EarthSearchPathMapper,
+    EarthSearchC1PathMapper,
+    SinergisePathMapper,
+)
+from mapchete_eo.platforms.sentinel2.metadata_parser.s2metadata import S2Metadata
+from mapchete_eo.search.s2_mgrs import S2Tile
+
+
+# mapper functions decorated with metadata to have driver decide which one to apply when #
+##########################################################################################
+
+
+@maps_item_id(from_collections=["EarthSearch", "EarthSearch_legacy"])
+def earthsearch_id_mapper(item: Item) -> Item:
+    item.id = item.properties["s2:product_uri"].rstrip(".SAFE")
+    return item
+
+
+@maps_stac_metadata(from_collections=["EarthSearch"], to_data_archives=["AWSCOG"])
+def earthsearch_assets_paths_mapper(item: Item) -> Item:
+    """Nothing to do here as paths match catalog."""
+    return item
+
+
+@creates_s2metadata(from_collections=["EarthSearch"], to_metadata_archives=["roda"])
+def earthsearch_to_s2metadata(item: Item) -> S2Metadata:
+    return S2Metadata.from_stac_item(
+        item,
+        path_mapper=EarthSearchC1PathMapper(
+            MPath(item.assets["granule_metadata"].href)
+        ),
+        processing_baseline_field="s2:processing_baseline",
+    )
+
+
+@creates_s2metadata(
+    from_collections=["EarthSearch_legacy"], to_metadata_archives=["roda"]
+)
+def earthsearch_legacy_to_s2metadata(item: Item) -> S2Metadata:
+    return S2Metadata.from_stac_item(
+        item,
+        path_mapper=EarthSearchPathMapper(MPath(item.assets["granule_metadata"].href)),
+        boa_offset_field="earthsearch:boa_offset_applied",
+        processing_baseline_field="s2:processing_baseline",
+    )
+
+
+@maps_item_id(from_collections=["CDSE"])
+def plain_id_mapper(item: Item) -> Item:
+    return item
+
+
+CDSE_ASSET_NAME_MAPPING = {
+    "AOT_10m": "aot",
+    "B01_20m": "coastal",
+    "B02_10m": "blue",
+    "B03_10m": "green",
+    "B04_10m": "red",
+    "B05_20m": "rededge1",
+    "B06_20m": "rededge2",
+    "B07_20m": "rededge3",
+    "B08_10m": "nir",
+    "B09_60m": "nir09",
+    "B11_20m": "swir16",
+    "B12_20m": "swir22",
+    "B8A_20m": "nir08",
+    "SCL_20m": "scl",
+    "TCI_10m": "visual",
+    "WVP_10m": "wvp",
+}
+
+
+@maps_stac_metadata(from_collections=["CDSE"])
+def cdse_asset_names(item: Item) -> Item:
+    new_assets = {}
+    for asset_name, asset in item.assets.items():
+        if asset_name in CDSE_ASSET_NAME_MAPPING:
+            asset_name = CDSE_ASSET_NAME_MAPPING[asset_name]
+        new_assets[asset_name] = asset
+
+    item.assets = new_assets
+
+    item.properties["s2:datastrip_id"] = item.properties.get("eopf:datastrip_id")
+    return item
+
+
+@maps_stac_metadata(from_collections=["CDSE"], to_data_archives=["AWSJP2"])
+def map_cdse_paths_to_jp2_archive(item: Item) -> Item:
+    """
+    CSDE has the following assets:
+    AOT_10m, AOT_20m, AOT_60m, B01_20m, B01_60m, B02_10m, B02_20m, B02_60m, B03_10m, B03_20m,
+    B03_60m, B04_10m, B04_20m, B04_60m, B05_20m, B05_60m, B06_20m, B06_60m, B07_20m, B07_60m,
+    B08_10m, B09_60m, B11_20m, B11_60m, B12_20m, B12_60m, B8A_20m, B8A_60m, Product, SCL_20m,
+    SCL_60m, TCI_10m, TCI_20m, TCI_60m, WVP_10m, WVP_20m, WVP_60m, thumbnail, safe_manifest,
+    granule_metadata, inspire_metadata, product_metadata, datastrip_metadata
+
+    sample path for AWS JP2:
+    s3://sentinel-s2-l2a/tiles/51/K/XR/2020/7/31/0/R10m/
+    """
+    if item.datetime is None:
+        raise ValueError(f"product {item.get_self_href()} does not have a timestamp")
+    path_base_scheme = "s3://sentinel-s2-l2a/tiles/{utm_zone}/{latitude_band}/{grid_square}/{year}/{month}/{day}/{count}"
+    s2tile = S2Tile.from_grid_code(item.properties["grid:code"])
+    product_basepath = MPath(
+        path_base_scheme.format(
+            utm_zone=int(s2tile.utm_zone),
+            latitude_band=s2tile.latitude_band,
+            grid_square=s2tile.grid_square,
+            year=item.datetime.year,
+            month=item.datetime.month,
+            day=item.datetime.day,
+            count=0,  # TODO: get count dynamically from metadata
+        )
+    )
+    new_assets = {}
+    for asset_name, asset in item.assets.items():
+        # ignore these assets
+        if asset_name in [
+            "Product",
+            "safe_manifest",
+            "product_metadata",
+            "inspire_metadata",
+            "datastrip_metadata",
+        ]:
+            continue
+        # set thumbnnail
+        elif asset_name == "thumbnail":
+            asset.href = str(product_basepath / "R60m" / "TCI.jp2")
+        # point to proper metadata
+        elif asset_name == "granule_metadata":
+            asset.href = str(product_basepath / "metadata.xml")
+        # change band asset names and point to their new locations
+        elif asset_name in CDSE_ASSET_NAME_MAPPING:
+            name, resolution = asset_name.split("_")
+            asset.href = product_basepath / f"R{resolution}" / f"{name}.jp2"
+            asset_name = CDSE_ASSET_NAME_MAPPING[asset_name]
+        else:
+            continue
+        new_assets[asset_name] = asset
+
+    item.assets = new_assets
+
+    return item
+
+
+@creates_s2metadata(from_collections=["CDSE"], to_metadata_archives=["CDSE"])
+def cdse_s2metadata(item: Item) -> S2Metadata:
+    return S2Metadata.from_stac_item(
+        item,
+        path_mapper=CDSEPathMapper(MPath(item.assets["granule_metadata"].href)),
+        processing_baseline_field="processing:version",
+    )
+
+
+@creates_s2metadata(from_collections=["CDSE"], to_metadata_archives=["roda"])
+def cdse_to_roda_s2metadata(item: Item) -> S2Metadata:
+    return S2Metadata.from_stac_item(
+        item,
+        path_mapper=SinergisePathMapper(MPath(item.assets["granule_metadata"].href)),
+        processing_baseline_field="processing:version",
+    )