mapchete-eo 2025.7.0__py2.py3-none-any.whl

mapchete_eo/product.py

@@ -0,0 +1,278 @@
+from __future__ import annotations
+
+import logging
+from typing import Any, List, Literal, Optional, Set
+
+import numpy as np
+import numpy.ma as ma
+import pystac
+import xarray as xr
+from mapchete import Timer
+from mapchete.io.raster import ReferencedRaster
+from mapchete.path import MPath, MPathLike
+from mapchete.protocols import GridProtocol
+from mapchete.types import Bounds, NodataVals
+from numpy.typing import DTypeLike
+from rasterio.enums import Resampling
+from shapely.geometry import shape
+
+from mapchete_eo.array.convert import to_dataarray
+from mapchete_eo.io import get_item_property, item_to_np_array
+from mapchete_eo.protocols import EOProductProtocol
+from mapchete_eo.settings import mapchete_eo_settings
+from mapchete_eo.types import BandLocation
+
+logger = logging.getLogger(__name__)
+
+
+class EOProduct(EOProductProtocol):
+    """Wrapper class around a pystac.Item which provides read functions."""
+
+    default_dtype: DTypeLike = np.uint16
+
+    def __init__(self, item: pystac.Item):
+        self.item_dict = item.to_dict()
+        self.__geo_interface__ = self.item.geometry
+        self.bounds = Bounds.from_inp(shape(self))
+        self.crs = mapchete_eo_settings.default_catalog_crs
+
+    def __repr__(self):
+        return f"<EOProduct product_id={self.item.id}>"
+
+    def clear_cached_data(self):
+        pass
+
+    @property
+    def item(self) -> pystac.Item:
+        return pystac.Item.from_dict(self.item_dict)
+
+    @classmethod
+    def from_stac_item(self, item: pystac.Item, **kwargs) -> EOProduct:
+        return EOProduct(item)
+
+    def get_mask(self) -> ReferencedRaster: ...
+
+    def read(
+        self,
+        assets: Optional[List[str]] = None,
+        eo_bands: Optional[List[str]] = None,
+        grid: Optional[GridProtocol] = None,
+        resampling: Resampling = Resampling.nearest,
+        nodatavals: NodataVals = None,
+        x_axis_name: str = "x",
+        y_axis_name: str = "y",
+        raise_empty: bool = True,
+        **kwargs,
+    ) -> xr.Dataset:
+        """Read bands and assets into xarray."""
+        # developer info: all fancy stuff for special platforms like Sentinel-2
+        # should be implemented in the respective read_np_array() methods which get
+        # called by this method. No need to apply masks etc. here too.
+        if isinstance(nodatavals, list):
+            nodataval = nodatavals[0]
+        elif isinstance(nodatavals, float):
+            nodataval = nodatavals
+        else:
+            nodataval = nodatavals
+
+        assets = assets or []
+        eo_bands = eo_bands or []
+        data_var_names = assets or eo_bands
+        return xr.Dataset(
+            data_vars={
+                data_var_name: to_dataarray(
+                    asset_arr,
+                    x_axis_name=x_axis_name,
+                    y_axis_name=y_axis_name,
+                    name=data_var_name,
+                    attrs=dict(item_id=self.item.id),
+                )
+                for asset_arr, data_var_name in zip(
+                    self.read_np_array(
+                        assets=assets,
+                        eo_bands=eo_bands,
+                        grid=grid,
+                        resampling=resampling,
+                        nodatavals=nodatavals,
+                        raise_empty=raise_empty,
+                        **kwargs,
+                    ),
+                    data_var_names,
+                )
+            },
+            coords={},
+            attrs=dict(self.item.properties, id=self.item.id, _FillValue=nodataval),
+        )
+
+    def read_np_array(
+        self,
+        assets: Optional[List[str]] = None,
+        eo_bands: Optional[List[str]] = None,
+        grid: Optional[GridProtocol] = None,
+        resampling: Resampling = Resampling.nearest,
+        nodatavals: NodataVals = None,
+        raise_empty: bool = True,
+        apply_offset: bool = True,
+        apply_scale: bool = False,
+        **kwargs,
+    ) -> ma.MaskedArray:
+        assets = assets or []
+        eo_bands = eo_bands or []
+        bands = assets or eo_bands
+        logger.debug("%s: reading assets %s over %s", self, bands, grid)
+        with Timer() as t:
+            out = item_to_np_array(
+                self.item,
+                self.assets_eo_bands_to_band_locations(assets, eo_bands),
+                grid=grid,
+                resampling=resampling,
+                nodatavals=nodatavals,
+                raise_empty=raise_empty,
+                apply_offset=apply_offset,
+            )
+        logger.debug("%s: read in %s", self, t)
+        return out
+
+    def empty_array(
+        self,
+        count: int,
+        grid: GridProtocol,
+        fill_value: int = 0,
+        dtype: Optional[DTypeLike] = None,
+    ) -> ma.MaskedArray:
+        shape = (count, *grid.shape)
+        dtype = dtype or self.default_dtype
+        return ma.MaskedArray(
+            data=np.full(shape, fill_value=fill_value, dtype=dtype),
+            mask=np.ones(shape, dtype=bool),
+            fill_value=fill_value,
+        )
+
+    def get_property(self, property: str) -> Any:
+        return get_item_property(self.item, property)
+
+    def eo_bands_to_band_location(self, eo_bands: List[str]) -> List[BandLocation]:
+        return eo_bands_to_band_locations(self.item, eo_bands)
+
+    def assets_eo_bands_to_band_locations(
+        self,
+        assets: Optional[List[str]] = None,
+        eo_bands: Optional[List[str]] = None,
+    ) -> List[BandLocation]:
+        assets = assets or []
+        eo_bands = eo_bands or []
+        if assets and eo_bands:
+            raise ValueError("assets and eo_bands cannot be provided at the same time")
+        if assets:
+            return [BandLocation(asset_name=asset) for asset in assets]
+        elif eo_bands:
+            return self.eo_bands_to_band_location(eo_bands)
+        else:
+            raise ValueError("assets or eo_bands have to be provided")
+
+
+def eo_bands_to_band_locations(
+    item: pystac.Item,
+    eo_bands: List[str],
+    role: Literal["data", "reflectance", "visual"] = "data",
+) -> List[BandLocation]:
+    """
+    Find out location (asset and band index) of EO band.
+    """
+    return [find_eo_band(item, eo_band, role=role) for eo_band in eo_bands]
+
+
+def find_eo_band(
+    item: pystac.Item,
+    eo_band_name: str,
+    role: Literal["data", "reflectance", "visual"] = "data",
+) -> BandLocation:
+    """
+    Tries to find the location of the most appropriate band using the EO band name.
+
+    This function looks into all assets and all eo bands for the given name and role.
+    """
+    results = []
+    for asset_name, asset in item.assets.items():
+        # search in eo:bands and alternatively in bands for eo:common_name
+        for band_index, band_info in enumerate(
+            asset.extra_fields.get("eo:bands", asset.extra_fields.get("bands", [])), 1
+        ):
+            if (
+                # if name matches eo band name
+                (
+                    eo_band_name == band_info.get("name")
+                    or eo_band_name == band_info.get("eo:common_name")
+                )
+                # if role is given, make sure it matches with desired role
+                and (asset.roles is None or role in asset.roles)
+            ):
+                results.append(
+                    BandLocation.from_asset(
+                        name=asset_name,
+                        band_index=band_index,
+                        asset=asset,
+                    )
+                )
+
+    if len(results) == 0:
+        raise KeyError(f"EO band {eo_band_name} not found in item assets")
+
+    elif len(results) == 1:
+        return results[0]
+
+    # if results are ambiguous, further filter them
+    else:
+        # only use locations which seem to have the original resolution
+        for matches in [_asset_name_equals_eo_name, _is_original_sampling]:
+            filtered_results = [
+                band_location for band_location in results if matches(band_location)
+            ]
+            if len(filtered_results) == 1:
+                return filtered_results[0]
+        else:  # pragma: no cover
+            raise ValueError(
+                f"EO band '{eo_band_name}' found in multiple assets: {', '.join(map(str, results))}"
+            )
+
+
+def _asset_name_equals_eo_name(band_location: BandLocation) -> bool:
+    return band_location.asset_name == band_location.eo_band_name
+
+
+def _is_original_sampling(band_location: BandLocation) -> bool:
+    return band_location.roles == [] or "sampling:original" in band_location.roles
+
+
+def add_to_blacklist(path: MPathLike, blacklist: Optional[MPath] = None) -> None:
+    blacklist = blacklist or mapchete_eo_settings.blacklist
+
+    if blacklist is None:
+        return
+
+    blacklist = MPath.from_inp(blacklist)
+
+    path = MPath.from_inp(path)
+
+    # make sure paths stay unique
+    if str(path) not in blacklist_products(blacklist):
+        logger.debug("add path %s to blacklist", str(path))
+        try:
+            with blacklist.open("a") as dst:
+                dst.write(f"{path}\n")
+        except FileNotFoundError:
+            with blacklist.open("w") as dst:
+                dst.write(f"{path}\n")
+
+
+def blacklist_products(blacklist: Optional[MPathLike] = None) -> Set[str]:
+    blacklist = blacklist or mapchete_eo_settings.blacklist
+    if blacklist is None:
+        raise ValueError("no blacklist is defined")
+    blacklist = MPath.from_inp(blacklist)
+
+    try:
+        return set(blacklist.read_text().splitlines())
+    except FileNotFoundError:
+        logger.debug("%s does not exist, returning empty set", str(blacklist))
+        return set()

mapchete_eo/protocols.py

@@ -0,0 +1,56 @@
+from __future__ import annotations
+
+from typing import Any, Dict, List, Optional, Protocol
+
+import numpy.ma as ma
+import pystac
+import xarray as xr
+from mapchete.protocols import GridProtocol
+from mapchete.types import Bounds, NodataVals
+from rasterio.crs import CRS
+from rasterio.enums import Resampling
+
+from mapchete_eo.types import DateTimeLike
+from mapchete.io.raster import ReferencedRaster
+
+
+class EOProductProtocol(Protocol):
+    bounds: Bounds
+    crs: CRS
+    __geo_interface__: Optional[Dict[str, Any]]
+
+    @classmethod
+    def from_stac_item(self, item: pystac.Item, **kwargs) -> EOProductProtocol: ...
+
+    def get_mask(self) -> ReferencedRaster: ...
+
+    def read(
+        self,
+        assets: Optional[List[str]] = None,
+        eo_bands: Optional[List[str]] = None,
+        grid: Optional[GridProtocol] = None,
+        resampling: Resampling = Resampling.nearest,
+        nodatavals: NodataVals = None,
+        x_axis_name: str = "x",
+        y_axis_name: str = "y",
+        **kwargs,
+    ) -> xr.Dataset: ...
+
+    def read_np_array(
+        self,
+        assets: Optional[List[str]] = None,
+        eo_bands: Optional[List[str]] = None,
+        grid: Optional[GridProtocol] = None,
+        resampling: Resampling = Resampling.nearest,
+        nodatavals: NodataVals = None,
+        **kwargs,
+    ) -> ma.MaskedArray: ...
+
+    def get_property(self, property: str) -> Any: ...
+
+    @property
+    def item(self) -> pystac.Item: ...
+
+
+class DateTimeProtocol(Protocol):
+    datetime: DateTimeLike

mapchete_eo/search/__init__.py

@@ -0,0 +1,14 @@
+"""
+A catalog is an instance with a specific endpoint and a specific collection.
+
+The catalog class aims to abstract product search as well as homogenization
+of product metadata.
+
+It helps the InputData class to find the input products and their metadata.
+"""
+
+from mapchete_eo.search.stac_search import STACSearchCatalog
+from mapchete_eo.search.stac_static import STACStaticCatalog
+from mapchete_eo.search.utm_search import UTMSearchCatalog
+
+__all__ = ["STACSearchCatalog", "STACStaticCatalog", "UTMSearchCatalog"]

mapchete_eo/search/base.py

@@ -0,0 +1,222 @@
+import json
+import logging
+from abc import ABC, abstractmethod
+from typing import Any, Callable, Dict, Generator, List, Optional, Type, Union
+
+from pydantic import BaseModel
+from pystac import Item, Catalog, CatalogType, Extent
+from mapchete.path import MPath, MPathLike
+from mapchete.types import Bounds
+from pystac.collection import Collection
+from pystac.stac_io import DefaultStacIO
+from pystac_client import Client
+from pystac_client.stac_api_io import StacApiIO
+from rasterio.profiles import Profile
+from shapely.geometry.base import BaseGeometry
+
+from mapchete_eo.io.assets import get_assets, get_metadata_assets
+from mapchete_eo.types import TimeRange
+
+logger = logging.getLogger(__name__)
+
+
+class FSSpecStacIO(StacApiIO):
+    """Custom class which allows I/O operations on object storage."""
+
+    def read_text(self, source: MPathLike, *args, **kwargs) -> str:
+        return MPath.from_inp(source).read_text()
+
+    def write_text(self, dest: MPathLike, txt: str, *args, **kwargs) -> None:
+        path = MPath.from_inp(dest)
+        if not path.parent.exists():
+            path.parent.makedirs(exist_ok=True)
+        with path.open("w") as dst:
+            return dst.write(txt)
+
+    # TODO: investigate in pystac why this has to be a staticmethod
+    @staticmethod
+    def save_json(dest: MPathLike, json_dict: dict, *args, **kwargs) -> None:
+        path = MPath.from_inp(dest)
+        if not path.parent.exists():
+            path.parent.makedirs(exist_ok=True)
+        with path.open("w") as dst:
+            return dst.write(json.dumps(json_dict, indent=2))
+
+
+class CatalogSearcher(ABC):
+    """
+    This class serves as a bridge between an Archive and a catalog implementation.
+    """
+
+    eo_bands: List[str]
+    id: str
+    description: str
+    stac_extensions: List[str]
+    collections: List[str]
+    config_cls: Type[BaseModel]
+
+    @abstractmethod
+    def search(
+        self,
+        time: Optional[Union[TimeRange, List[TimeRange]]] = None,
+        bounds: Optional[Bounds] = None,
+        area: Optional[BaseGeometry] = None,
+        search_kwargs: Optional[Dict[str, Any]] = None,
+    ) -> Generator[Item, None, None]: ...
+
+
+class StaticCatalogWriterMixin(CatalogSearcher):
+    client: Client
+    id: str
+    description: str
+    stac_extensions: List[str]
+
+    @abstractmethod
+    def get_collections(self) -> List[Collection]:  # pragma: no cover
+        ...
+
+    def write_static_catalog(
+        self,
+        output_path: MPathLike,
+        bounds: Optional[Bounds] = None,
+        area: Optional[BaseGeometry] = None,
+        time: Optional[TimeRange] = None,
+        search_kwargs: Optional[Dict[str, Any]] = None,
+        name: Optional[str] = None,
+        description: Optional[str] = None,
+        assets: Optional[List[str]] = None,
+        assets_dst_resolution: Union[None, float, int] = None,
+        assets_convert_profile: Optional[Profile] = None,
+        copy_metadata: bool = False,
+        metadata_parser_classes: Optional[tuple] = None,
+        overwrite: bool = False,
+        stac_io: DefaultStacIO = FSSpecStacIO(),
+        progress_callback: Optional[Callable] = None,
+    ) -> MPath:
+        """Dump static version of current items."""
+        output_path = MPath.from_inp(output_path)
+        assets = assets or []
+        # initialize catalog
+        catalog_json = output_path / "catalog.json"
+        if catalog_json.exists():
+            logger.debug("open existing catalog %s", str(catalog_json))
+            client = Client.from_file(catalog_json)
+            # catalog = pystac.Catalog.from_file(catalog_json)
+            existing_collections = list(client.get_collections())
+        else:
+            existing_collections = []
+        catalog = Catalog(
+            name or f"{self.id}",
+            description or f"Static subset of {self.description}",
+            stac_extensions=self.stac_extensions,
+            href=str(catalog_json),
+            catalog_type=CatalogType.SELF_CONTAINED,
+        )
+        src_items = list(
+            self.search(
+                time=time, bounds=bounds, area=area, search_kwargs=search_kwargs
+            )
+        )
+        for collection in self.get_collections():
+            # collect all items and download assets if required
+            items: List[Item] = []
+            item_ids = set()
+            for n, item in enumerate(src_items, 1):
+                logger.debug("found item %s", item)
+                item = item.clone()
+                if assets:
+                    logger.debug("get assets %s", assets)
+                    item = get_assets(
+                        item,
+                        assets,
+                        output_path / collection.id / item.id,
+                        resolution=assets_dst_resolution,
+                        convert_profile=assets_convert_profile,
+                        overwrite=overwrite,
+                        ignore_if_exists=True,
+                    )
+                if copy_metadata:
+                    item = get_metadata_assets(
+                        item,
+                        output_path / collection.id / item.id,
+                        metadata_parser_classes=metadata_parser_classes,
+                        resolution=assets_dst_resolution,
+                        convert_profile=assets_convert_profile,
+                        overwrite=overwrite,
+                    )
+                # this has to be set to None, otherwise pystac will mess up the asset paths
+                # after normalizing
+                item.set_self_href(None)
+
+                items.append(item)
+                item_ids.add(item.id)
+
+                if progress_callback:
+                    progress_callback(n=n, total=len(src_items))
+
+            for existing_collection in existing_collections:
+                if existing_collection.id == collection.id:
+                    logger.debug("try to find unregistered items in collection")
+                    collection_root_path = MPath.from_inp(
+                        existing_collection.get_self_href()
+                    ).parent
+                    for subpath in collection_root_path.ls():
+                        if subpath.is_directory():
+                            try:
+                                item = Item.from_file(
+                                    subpath / subpath.with_suffix(".json").name
+                                )
+                                if item.id not in item_ids:
+                                    logger.debug(
+                                        "add existing item with id %s", item.id
+                                    )
+                                    items.append(item)
+                                    item_ids.add(item.id)
+                            except FileNotFoundError:
+                                pass
+                    break
+            # create collection and copy metadata
+            logger.debug("create new collection")
+            out_collection = Collection(
+                id=collection.id,
+                extent=Extent.from_items(items),
+                description=collection.description,
+                title=collection.title,
+                stac_extensions=collection.stac_extensions,
+                license=collection.license,
+                keywords=collection.keywords,
+                providers=collection.providers,
+                summaries=collection.summaries,
+                extra_fields=collection.extra_fields,
+                catalog_type=CatalogType.SELF_CONTAINED,
+            )
+
+            # finally, add all items to collection
+            for item in items:
+                out_collection.add_item(item)
+
+            out_collection.update_extent_from_items()
+
+            catalog.add_child(out_collection)
+
+        logger.debug("write catalog to %s", output_path)
+        catalog.normalize_hrefs(str(output_path))
+        catalog.make_all_asset_hrefs_relative()
+        catalog.save(dest_href=str(output_path), stac_io=stac_io)
+
+        return catalog_json
+
+
+def filter_items(
+    items: Generator[Item, None, None],
+    cloud_cover_field: str = "eo:cloud_cover",
+    max_cloud_cover: float = 100.0,
+) -> Generator[Item, None, None]:
+    """
+    Only for cloudcover now, this can and should be adapted for filter field and value
+    the field and value for the item filter would be defined in search.config.py corresponding configs
+    and passed down to the individual search approaches via said config and this Function.
+    """
+    for item in items:
+        if item.properties.get(cloud_cover_field, 0.0) <= max_cloud_cover:
+            yield item

mapchete_eo/search/config.py

@@ -0,0 +1,42 @@
+from typing import Optional
+
+from mapchete.path import MPath, MPathLike
+from pydantic import BaseModel
+
+
+class StacSearchConfig(BaseModel):
+    max_cloud_cover: float = 100.0
+    catalog_chunk_threshold: int = 10_000
+    catalog_chunk_zoom: int = 5
+    catalog_pagesize: int = 100
+    footprint_buffer: float = 0
+
+
+class StacStaticConfig(BaseModel):
+    max_cloud_cover: float = 100.0
+
+
+class UTMSearchConfig(BaseModel):
+    max_cloud_cover: float = 100.0
+
+    sinergise_aws_collections: dict = dict(
+        S2_L2A=dict(
+            id="sentinel-s2-l2a",
+            path=MPath(
+                "https://sentinel-s2-l2a-stac.s3.amazonaws.com/sentinel-s2-l2a.json"
+            ),
+        ),
+        S2_L1C=dict(
+            id="sentinel-s2-l1c",
+            path=MPath(
+                "https://sentinel-s2-l1c-stac.s3.amazonaws.com/sentinel-s2-l1c.json"
+            ),
+        ),
+        S1_GRD=dict(
+            id="sentinel-s1-l1c",
+            path=MPath(
+                "https://sentinel-s1-l1c-stac.s3.amazonaws.com/sentinel-s1-l1c.json"
+            ),
+        ),
+    )
+    search_index: Optional[MPathLike] = None