mapchete-eo 2025.7.0__py2.py3-none-any.whl

mapchete_eo/platforms/sentinel2/path_mappers/__init__.py

@@ -0,0 +1,29 @@
+from mapchete_eo.platforms.sentinel2.path_mappers.base import S2PathMapper
+from mapchete_eo.platforms.sentinel2.path_mappers.earthsearch import (
+    EarthSearchPathMapper,
+)
+from mapchete_eo.platforms.sentinel2.path_mappers.metadata_xml import XMLMapper
+from mapchete_eo.platforms.sentinel2.path_mappers.sinergise import SinergisePathMapper
+
+
+def default_path_mapper_guesser(
+    url: str,
+    **kwargs,
+) -> S2PathMapper:
+    """Guess S2PathMapper based on URL.
+
+    If a new path mapper is added in this module, it should also be added to this function
+    in order to be detected.
+    """
+    if url.startswith(
+        ("https://roda.sentinel-hub.com/sentinel-s2-l2a/", "s3://sentinel-s2-l2a/")
+    ) or url.startswith(
+        ("https://roda.sentinel-hub.com/sentinel-s2-l1c/", "s3://sentinel-s2-l1c/")
+    ):
+        return SinergisePathMapper(url, **kwargs)
+    elif url.startswith(
+        "https://sentinel-cogs.s3.us-west-2.amazonaws.com/sentinel-s2-l2a-cogs/"
+    ):
+        return EarthSearchPathMapper(url, **kwargs)
+    else:
+        return XMLMapper(url, **kwargs)

mapchete_eo/platforms/sentinel2/path_mappers/base.py

@@ -0,0 +1,56 @@
+from abc import ABC, abstractmethod
+
+from mapchete.path import MPath
+
+from mapchete_eo.platforms.sentinel2.processing_baseline import ProcessingBaseline
+from mapchete_eo.platforms.sentinel2.types import (
+    BandQI,
+    L2ABand,
+    ProductQI,
+    ProductQIMaskResolution,
+)
+
+
+class S2PathMapper(ABC):
+    """
+    Abstract class to help mapping asset paths from metadata.xml to their
+    locations of various data archives.
+
+    This is mainly used for additional data like QI masks.
+    """
+
+    # All available bands for Sentinel-2 Level 2A.
+    _bands = [band.name for band in L2ABand]
+
+    processing_baseline: ProcessingBaseline
+
+    @abstractmethod
+    def product_qi_mask(
+        self,
+        qi_mask: ProductQI,
+        resolution: ProductQIMaskResolution = ProductQIMaskResolution["60m"],
+    ) -> MPath: ...
+
+    @abstractmethod
+    def classification_mask(self) -> MPath: ...
+
+    @abstractmethod
+    def cloud_probability_mask(
+        self, resolution: ProductQIMaskResolution = ProductQIMaskResolution["60m"]
+    ) -> MPath: ...
+
+    @abstractmethod
+    def snow_probability_mask(
+        self, resolution: ProductQIMaskResolution = ProductQIMaskResolution["60m"]
+    ) -> MPath: ...
+
+    @abstractmethod
+    def band_qi_mask(self, qi_mask: BandQI, band: L2ABand) -> MPath: ...
+
+    @abstractmethod
+    def technical_quality_mask(self, band: L2ABand) -> MPath: ...
+
+    @abstractmethod
+    def detector_footprints(self, band: L2ABand) -> MPath: ...
+
+    def clear_cached_data(self) -> None: ...

mapchete_eo/platforms/sentinel2/path_mappers/earthsearch.py

@@ -0,0 +1,34 @@
+from mapchete.path import MPath
+
+from mapchete_eo.platforms.sentinel2.path_mappers.sinergise import SinergisePathMapper
+from mapchete_eo.platforms.sentinel2.processing_baseline import ProcessingBaseline
+
+
+class EarthSearchPathMapper(SinergisePathMapper):
+    """
+    The COG archive maintained by E84 and covered by EarthSearch does not hold additional data
+    such as the GML files. This class maps the metadata masks to the current EarthSearch product.
+
+    e.g.:
+    B01 detector footprints: s3://sentinel-s2-l2a/tiles/51/K/XR/2020/7/31/0/qi/MSK_DETFOO_B01.gml
+    Cloud masks: s3://sentinel-s2-l2a/tiles/51/K/XR/2020/7/31/0/qi/MSK_CLOUDS_B00.gml
+
+    newer products however:
+    B01 detector footprints: s3://sentinel-s2-l2a/tiles/51/K/XR/2022/6/6/0/qi/DETFOO_B01.jp2
+    no vector cloudmasks available anymore
+    """
+
+    def __init__(
+        self,
+        metadata_xml: MPath,
+        alternative_metadata_baseurl: str = "sentinel-s2-l2a",
+        protocol: str = "s3",
+        baseline_version: str = "04.00",
+        **kwargs,
+    ):
+        basedir = metadata_xml.parent
+        self._path = (basedir / "tileinfo_metadata.json").read_json()["path"]
+        self._utm_zone, self._latitude_band, self._grid_square = basedir.elements[-6:-3]
+        self._baseurl = alternative_metadata_baseurl
+        self._protocol = protocol
+        self.processing_baseline = ProcessingBaseline.from_version(baseline_version)

mapchete_eo/platforms/sentinel2/path_mappers/metadata_xml.py

@@ -0,0 +1,135 @@
+"""
+A path mapper maps from an metadata XML file to additional metadata
+on a given archive or a local SAFE file.
+"""
+
+import logging
+from xml.etree.ElementTree import Element
+from functools import cached_property
+from typing import Optional
+
+from mapchete.path import MPath
+
+from mapchete_eo.io import open_xml
+from mapchete_eo.platforms.sentinel2.path_mappers.base import S2PathMapper
+from mapchete_eo.platforms.sentinel2.processing_baseline import ProcessingBaseline
+from mapchete_eo.platforms.sentinel2.types import (
+    BandQI,
+    L2ABand,
+    ProductQI,
+    ProductQIMaskResolution,
+)
+
+logger = logging.getLogger(__name__)
+
+
+class XMLMapper(S2PathMapper):
+    def __init__(
+        self, metadata_xml: MPath, xml_root: Optional[Element] = None, **kwargs
+    ):
+        self.metadata_xml = metadata_xml
+        self._cached_xml_root = xml_root
+        self._metadata_dir = metadata_xml.parent
+
+    def clear_cached_data(self):
+        if self._cached_xml_root is not None:
+            logger.debug("clear XMLMapper xml cache")
+            self._cached_xml_root.clear()
+            self._cached_xml_root = None
+
+    @property
+    def xml_root(self) -> Element:
+        if self._cached_xml_root is None:
+            self._cached_xml_root = open_xml(self.metadata_xml)
+        return self._cached_xml_root
+
+    @cached_property
+    def processing_baseline(self):
+        # try to guess processing baseline from product id
+        def _get_version(tag="TILE_ID"):
+            product_id = next(self.xml_root.iter(tag)).text
+            appendix = product_id.split("_")[-1]
+            if appendix.startswith("N"):
+                return appendix.lstrip("N")
+
+        version = _get_version()
+        try:
+            return ProcessingBaseline.from_version(version)
+        except Exception:  # pragma: no cover
+            # try use L1C product version as fallback
+            # we don't need to test this because HOPEFULLY we won't be confronted
+            # with such data
+            try:
+                l1c_version = _get_version("L1C_TILE_ID")
+            except StopIteration:
+                l1c_version = "02.06"
+            if l1c_version is not None:
+                return ProcessingBaseline.from_version(f"{l1c_version}")
+
+    def product_qi_mask(
+        self,
+        qi_mask: ProductQI,
+        resolution: ProductQIMaskResolution = ProductQIMaskResolution["60m"],
+    ) -> MPath:
+        """Determine product QI mask from metadata.xml."""
+        qi_mask_type = dict(self.processing_baseline.product_mask_types)[qi_mask]
+        for i in self.xml_root.iter():
+            if i.tag == "MASK_FILENAME" and i.get("type") == qi_mask_type:
+                path = self._metadata_dir / i.text
+                if qi_mask == ProductQI.classification:
+                    return path
+                else:
+                    if resolution.name in path.name:
+                        return path
+        else:
+            raise KeyError(f"no {qi_mask_type} with item found in metadata")
+
+    def classification_mask(self) -> MPath:
+        return self.product_qi_mask(ProductQI.classification)
+
+    def cloud_probability_mask(
+        self, resolution: ProductQIMaskResolution = ProductQIMaskResolution["60m"]
+    ) -> MPath:
+        return self.product_qi_mask(ProductQI.cloud_probability, resolution=resolution)
+
+    def snow_probability_mask(
+        self, resolution: ProductQIMaskResolution = ProductQIMaskResolution["60m"]
+    ) -> MPath:
+        return self.product_qi_mask(ProductQI.snow_probability, resolution=resolution)
+
+    def band_qi_mask(self, qi_mask: BandQI, band: L2ABand) -> MPath:
+        """Determine band QI mask from metadata.xml."""
+        if qi_mask.name not in dict(self.processing_baseline.band_mask_types).keys():
+            raise DeprecationWarning(
+                f"QI mask '{qi_mask}' not available for this product"
+            )
+        mask_types = set()
+        for masks in self.xml_root.iter("Pixel_Level_QI"):
+            if masks.get("geometry") == "FULL_RESOLUTION":
+                for mask_path in masks:
+                    qi_mask_type = dict(self.processing_baseline.band_mask_types)[
+                        qi_mask
+                    ]
+                    mask_type = mask_path.get("type")
+                    if mask_type:
+                        mask_types.add(mask_type)
+                        if mask_type == qi_mask_type:
+                            band_id = mask_path.get("bandId")
+                            if band_id is not None:
+                                band_idx = int(band_id)
+                                if band_idx == band.value:
+                                    return self._metadata_dir / mask_path.text
+                else:  # pragma: no cover
+                    raise KeyError(
+                        f"no {qi_mask_type} for band {band.name} not found in metadata: {', '.join(mask_types)}"
+                    )
+        else:  # pragma: no cover
+            raise KeyError(
+                f"no {qi_mask_type} not found in metadata: {', '.join(mask_types)}"
+            )
+
+    def technical_quality_mask(self, band: L2ABand) -> MPath:
+        return self.band_qi_mask(BandQI.technical_quality, band)
+
+    def detector_footprints(self, band: L2ABand) -> MPath:
+        return self.band_qi_mask(BandQI.detector_footprints, band)

mapchete_eo/platforms/sentinel2/path_mappers/sinergise.py

@@ -0,0 +1,105 @@
+from mapchete.path import MPath, MPathLike
+
+from mapchete_eo.platforms.sentinel2.path_mappers.base import S2PathMapper
+from mapchete_eo.platforms.sentinel2.processing_baseline import ProcessingBaseline
+from mapchete_eo.platforms.sentinel2.types import (
+    BandQI,
+    L2ABand,
+    ProductQI,
+    ProductQIMaskResolution,
+)
+
+
+class SinergisePathMapper(S2PathMapper):
+    """
+    Return true paths of product quality assets from the Sinergise S2 bucket.
+
+    e.g.:
+    B01 detector footprints: s3://sentinel-s2-l2a/tiles/51/K/XR/2020/7/31/0/qi/MSK_DETFOO_B01.gml
+    Cloud masks: s3://sentinel-s2-l2a/tiles/51/K/XR/2020/7/31/0/qi/MSK_CLOUDS_B00.gml
+
+    newer products however:
+    B01 detector footprints: s3://sentinel-s2-l2a/tiles/51/K/XR/2022/6/6/0/qi/DETFOO_B01.jp2
+    no vector cloudmasks available anymore
+    """
+
+    _PRE_0400_MASK_PATHS = {
+        ProductQI.classification: "MSK_CLOUDS_B00.gml",
+        ProductQI.cloud_probability: "CLD_{resolution}.jp2",  # are they really there?
+        ProductQI.snow_probability: "SNW_{resolution}.jp2",  # are they really there?
+        BandQI.detector_footprints: "MSK_DETFOO_{band_identifier}.gml",
+        BandQI.technical_quality: "MSK_TECQUA_{band_identifier}.gml",
+    }
+    _POST_0400_MASK_PATHS = {
+        ProductQI.classification: "CLASSI_B00.jp2",
+        ProductQI.cloud_probability: "CLD_{resolution}.jp2",
+        ProductQI.snow_probability: "SNW_{resolution}.jp2",
+        BandQI.detector_footprints: "DETFOO_{band_identifier}.jp2",
+        BandQI.technical_quality: "QUALIT_{band_identifier}.jp2",
+    }
+
+    def __init__(
+        self,
+        url: MPathLike,
+        bucket: str = "sentinel-s2-l2a",
+        protocol: str = "s3",
+        baseline_version: str = "04.00",
+        **kwargs,
+    ):
+        url = MPath.from_inp(url)
+        tileinfo_path = url.parent / "tileInfo.json"
+        self._path = MPath(
+            "/".join(tileinfo_path.elements[-9:-1]), **tileinfo_path._kwargs
+        )
+        self._utm_zone, self._latitude_band, self._grid_square = self._path.split("/")[
+            1:-4
+        ]
+        self._baseurl = bucket
+        self._protocol = protocol
+        self.processing_baseline = ProcessingBaseline.from_version(baseline_version)
+
+    def product_qi_mask(
+        self,
+        qi_mask: ProductQI,
+        resolution: ProductQIMaskResolution = ProductQIMaskResolution["60m"],
+    ) -> MPath:
+        """Determine product QI mask according to Sinergise bucket schema."""
+        if self.processing_baseline.version < "04.00":
+            mask_path = self._PRE_0400_MASK_PATHS[qi_mask]
+        else:
+            mask_path = self._POST_0400_MASK_PATHS[qi_mask]
+        key = f"{self._path}/qi/{mask_path.format(resolution=resolution.name)}"
+        return MPath.from_inp(f"{self._protocol}://{self._baseurl}/{key}")
+
+    def classification_mask(self) -> MPath:
+        return self.product_qi_mask(ProductQI.classification)
+
+    def cloud_probability_mask(
+        self, resolution: ProductQIMaskResolution = ProductQIMaskResolution["60m"]
+    ) -> MPath:
+        return self.product_qi_mask(ProductQI.cloud_probability, resolution=resolution)
+
+    def snow_probability_mask(
+        self, resolution: ProductQIMaskResolution = ProductQIMaskResolution["60m"]
+    ) -> MPath:
+        return self.product_qi_mask(ProductQI.snow_probability, resolution=resolution)
+
+    def band_qi_mask(self, qi_mask: BandQI, band: L2ABand) -> MPath:
+        """Determine product QI mask according to Sinergise bucket schema."""
+        try:
+            if self.processing_baseline.version < "04.00":
+                mask_path = self._PRE_0400_MASK_PATHS[qi_mask]
+            else:
+                mask_path = self._POST_0400_MASK_PATHS[qi_mask]
+        except KeyError:
+            raise DeprecationWarning(
+                f"'{qi_mask.name}' quality mask not found in this product"
+            )
+        key = f"{self._path}/qi/{mask_path.format(band_identifier=band.name)}"
+        return MPath.from_inp(f"{self._protocol}://{self._baseurl}/{key}")
+
+    def technical_quality_mask(self, band: L2ABand) -> MPath:
+        return self.band_qi_mask(BandQI.technical_quality, band)
+
+    def detector_footprints(self, band: L2ABand) -> MPath:
+        return self.band_qi_mask(BandQI.detector_footprints, band)

mapchete_eo/platforms/sentinel2/preprocessing_tasks.py

@@ -0,0 +1,26 @@
+import logging
+from typing import Optional, Union
+
+import pystac
+
+from mapchete_eo.exceptions import CorruptedProductMetadata
+from mapchete_eo.platforms.sentinel2.config import CacheConfig
+from mapchete_eo.platforms.sentinel2.product import S2Product
+from mapchete_eo.product import add_to_blacklist
+
+logger = logging.getLogger(__name__)
+
+
+def parse_s2_product(
+    item: pystac.Item,
+    cache_config: Optional[CacheConfig] = None,
+    cache_all: bool = False,
+) -> Union[S2Product, CorruptedProductMetadata]:
+    try:
+        s2product = S2Product.from_stac_item(
+            item, cache_config=cache_config, cache_all=cache_all
+        )
+    except CorruptedProductMetadata as exc:
+        add_to_blacklist(item.get_self_href())
+        return exc
+    return s2product

mapchete_eo/platforms/sentinel2/processing_baseline.py

@@ -0,0 +1,160 @@
+from dataclasses import dataclass
+from typing import Any, Optional, Union
+
+from pydantic import BaseModel
+
+
+class ProductMaskTypes(BaseModel):
+    """Mapping between mask type and respective metadata.xml type key."""
+
+    classification: Optional[str] = None
+    cloud_probability: Optional[str] = None
+    snow_probability: Optional[str] = None
+
+
+class BandMaskTypes(BaseModel):
+    """Mapping between band mask type and respective metadata.xml type key."""
+
+    technical_quality: Optional[str] = None
+    detector_footprints: Optional[str] = None
+    # deprecated since 04.00
+    # nodata: Optional[str] = None
+    # defect: Optional[str] = None
+    # saturated: Optional[str] = None
+
+
+class ItemMapping(BaseModel):
+    """Configuration of processing baseline keys in metadata.xml."""
+
+    product_mask_types: ProductMaskTypes
+    band_mask_types: BandMaskTypes
+    band_mask_extension: str
+
+
+# Available product mask types from PB 00.01 until 03.01.
+# "classification" mask was provided as GML, the other two as JP2.
+# Cloud probability and snow probability are available in two separate files for
+# 20m and 60m.
+pre_0400 = ItemMapping(
+    product_mask_types=ProductMaskTypes(
+        classification="MSK_CLOUDS",
+        cloud_probability="MSK_CLDPRB",
+        snow_probability="MSK_SNWPRB",
+    ),
+    band_mask_types=BandMaskTypes(
+        technical_quality="MSK_TECQUA",
+        detector_footprints="MSK_DETFOO",
+    ),
+    band_mask_extension="gml",
+)
+
+
+# Available product mask types from PB 04.00 onwards.
+# Cloud probability and snow probability are available in two separate files for
+# 20m and 60m.
+post_0400 = ItemMapping(
+    product_mask_types=ProductMaskTypes(
+        classification="MSK_CLASSI",
+        cloud_probability="MSK_CLDPRB",
+        snow_probability="MSK_SNWPRB",
+    ),
+    band_mask_types=BandMaskTypes(
+        technical_quality="MSK_QUALIT",
+        detector_footprints="MSK_DETFOO",
+    ),
+    band_mask_extension="jp2",
+)
+
+
+@dataclass
+class BaselineVersion:
+    """Helper for Processing Baseline versions."""
+
+    major: int
+    minor: int
+    level: str
+
+    @staticmethod
+    def from_string(version: str) -> "BaselineVersion":
+        major, minor = map(int, version.split("."))
+        if major < 2:
+            level = "L1C"
+        # everything below 02.06 is Level 1C
+        elif major == 2 and minor <= 6:
+            level = "L1C"
+        else:
+            level = "L2A"
+        return BaselineVersion(major, minor, level)
+
+    @staticmethod
+    def from_inp(inp: Union[str, "BaselineVersion"]) -> "BaselineVersion":
+        if isinstance(inp, str):
+            return BaselineVersion.from_string(inp)
+        elif isinstance(inp, BaselineVersion):
+            return inp
+        else:
+            raise TypeError(f"cannot generate BaselineVersion from input {inp}")
+
+    def __eq__(self, other: Any):
+        other = BaselineVersion.from_inp(other)
+        return self.major == other.major and self.minor == other.minor
+
+    def __lt__(self, other: Union[str, "BaselineVersion"]):
+        other = BaselineVersion.from_inp(other)
+        if self.major == other.major:
+            return self.minor < other.minor
+        else:
+            return self.major < other.major
+
+    def __le__(self, other: Union[str, "BaselineVersion"]):
+        other = BaselineVersion.from_inp(other)
+        if self.major == other.major:
+            return self.minor <= other.minor
+        else:
+            return self.major <= other.major
+
+    def __gt__(self, other: Union[str, "BaselineVersion"]):
+        other = BaselineVersion.from_inp(other)
+        if self.major == other.major:
+            return self.minor > other.minor
+        else:
+            return self.major > other.major
+
+    def __ge__(self, other: Union[str, "BaselineVersion"]):
+        if isinstance(other, str):
+            other = BaselineVersion.from_string(other)
+        if self.major == other.major:
+            return self.minor >= other.minor
+        else:
+            return self.major >= other.major
+
+    def __str__(self):
+        return f"{self.major:02}.{self.minor:02}"
+
+
+class ProcessingBaseline:
+    """Class which combines PB version and metadata.xml keys for QI masks."""
+
+    version: BaselineVersion
+    item_mapping: ItemMapping
+    product_mask_types: ProductMaskTypes
+    band_mask_types: BandMaskTypes
+    band_mask_extension: str
+
+    def __init__(self, version: BaselineVersion):
+        self.version = version
+        if self.version.major < 4:
+            self.item_mapping = pre_0400
+        else:
+            self.item_mapping = post_0400
+
+        self.product_mask_types = self.item_mapping.product_mask_types
+        self.band_mask_types = self.item_mapping.band_mask_types
+        self.band_mask_extension = self.item_mapping.band_mask_extension
+
+    @staticmethod
+    def from_version(version: Union[BaselineVersion, str]) -> "ProcessingBaseline":
+        if isinstance(version, BaselineVersion):
+            return ProcessingBaseline(version=version)
+        else:
+            return ProcessingBaseline(version=BaselineVersion.from_string(version))