mapchete-eo 2026.2.0__py3-none-any.whl

mapchete_eo/product.py

@@ -0,0 +1,323 @@
+from __future__ import annotations
+
+import logging
+from typing import Any, List, Literal, Optional, Set
+
+import numpy as np
+import numpy.ma as ma
+from pystac import Item
+import xarray as xr
+from mapchete import Timer
+from mapchete.io.raster import ReferencedRaster
+from mapchete.path import MPath, MPathLike
+from mapchete.protocols import GridProtocol
+from mapchete.types import Bounds, NodataVals
+from numpy.typing import DTypeLike
+from rasterio.enums import Resampling
+from shapely.geometry import shape
+
+from mapchete_eo.array.convert import to_dataarray
+from mapchete_eo.io import get_item_property, item_to_np_array
+from mapchete_eo.protocols import EOProductProtocol
+from mapchete_eo.settings import mapchete_eo_settings
+from mapchete_eo.types import BandLocation
+
+logger = logging.getLogger(__name__)
+
+
+class EOProduct(EOProductProtocol):
+    """
+    Wrapper class around a STAC Item which provides data reading capabilities.
+    """
+
+    id: str
+    default_dtype: DTypeLike = np.uint16
+    _item: Optional[Item] = None
+
+    def __init__(self, item: Item):
+        self.item_dict = item.to_dict()
+        self.__geo_interface__ = self.item.geometry
+        self.bounds = Bounds.from_inp(shape(self))
+        self.crs = mapchete_eo_settings.default_catalog_crs
+        self._item = None
+        self.id = item.id
+
+    def __repr__(self):
+        return f"<EOProduct product_id={self.item.id}>"
+
+    def clear_cached_data(self):
+        pass
+
+    @property
+    def item(self) -> Item:
+        if not self._item:
+            self._item = Item.from_dict(self.item_dict)
+        return self._item
+
+    @classmethod
+    def from_stac_item(self, item: Item, **kwargs) -> EOProduct:
+        return EOProduct(item)
+
+    def get_mask(self) -> ReferencedRaster: ...
+
+    def read(
+        self,
+        assets: Optional[List[str]] = None,
+        eo_bands: Optional[List[str]] = None,
+        grid: Optional[GridProtocol] = None,
+        resampling: Resampling = Resampling.nearest,
+        nodatavals: NodataVals = None,
+        x_axis_name: str = "x",
+        y_axis_name: str = "y",
+        raise_empty: bool = True,
+        **kwargs,
+    ) -> xr.Dataset:
+        """
+        Read bands and assets into an xarray.Dataset.
+
+        Args:
+            assets: List of asset names.
+            eo_bands: List of EO band names.
+            grid: Target grid protocol.
+            resampling: Resampling algorithm.
+            nodatavals: Custom nodata values.
+            x_axis_name: Name of X axis in output.
+            y_axis_name: Name of Y axis in output.
+            raise_empty: Raise exception if no data is found.
+
+        Returns:
+            xr.Dataset: Dataset with assets as data variables.
+        """
+        # developer info: all fancy stuff for special platforms like Sentinel-2
+        # should be implemented in the respective read_np_array() methods which get
+        # called by this method. No need to apply masks etc. here too.
+        if isinstance(nodatavals, list):
+            nodataval = nodatavals[0]
+        elif isinstance(nodatavals, float):
+            nodataval = nodatavals
+        else:
+            nodataval = nodatavals
+
+        assets = assets or []
+        eo_bands = eo_bands or []
+        data_var_names = assets or eo_bands
+        return xr.Dataset(
+            data_vars={
+                data_var_name: to_dataarray(
+                    asset_arr,
+                    x_axis_name=x_axis_name,
+                    y_axis_name=y_axis_name,
+                    name=data_var_name,
+                    attrs=dict(item_id=self.item.id),
+                )
+                for asset_arr, data_var_name in zip(
+                    self.read_np_array(
+                        assets=assets,
+                        eo_bands=eo_bands,
+                        grid=grid,
+                        resampling=resampling,
+                        nodatavals=nodatavals,
+                        raise_empty=raise_empty,
+                        **kwargs,
+                    ),
+                    data_var_names,
+                )
+            },
+            coords={},
+            attrs=dict(self.item.properties, id=self.item.id, _FillValue=nodataval),
+        )
+
+    def read_np_array(
+        self,
+        assets: Optional[List[str]] = None,
+        eo_bands: Optional[List[str]] = None,
+        grid: Optional[GridProtocol] = None,
+        resampling: Resampling = Resampling.nearest,
+        nodatavals: NodataVals = None,
+        raise_empty: bool = True,
+        apply_offset: bool = True,
+        **kwargs,
+    ) -> ma.MaskedArray:
+        """
+        Read assets or EO bands into a MaskedArray.
+
+        Args:
+            assets: List of asset names.
+            eo_bands: List of EO band names.
+            grid: Target grid.
+            resampling: Resampling method.
+            nodatavals: Nodata values.
+            raise_empty: Raise if empty.
+            apply_offset: Apply offset/scale metadata if present.
+
+        Returns:
+            ma.MaskedArray: Output array.
+        """
+        assets = assets or []
+        eo_bands = eo_bands or []
+        bands = assets or eo_bands
+        logger.debug("%s: reading assets %s over %s", self, bands, grid)
+        with Timer() as t:
+            out = item_to_np_array(
+                self.item,
+                self.assets_eo_bands_to_band_locations(assets, eo_bands),
+                grid=grid,
+                resampling=resampling,
+                nodatavals=nodatavals,
+                raise_empty=raise_empty,
+                apply_offset=apply_offset,
+            )
+        logger.debug("%s: read in %s", self, t)
+        return out
+
+    def empty_array(
+        self,
+        count: int,
+        grid: GridProtocol,
+        fill_value: int = 0,
+        dtype: Optional[DTypeLike] = None,
+    ) -> ma.MaskedArray:
+        shape = (count, *grid.shape)
+        dtype = dtype or self.default_dtype
+        return ma.MaskedArray(
+            data=np.full(shape, fill_value=fill_value, dtype=dtype),
+            mask=np.ones(shape, dtype=bool),
+            fill_value=fill_value,
+        )
+
+    def get_property(self, property: str) -> Any:
+        return get_item_property(self.item, property)
+
+    def eo_bands_to_band_location(self, eo_bands: List[str]) -> List[BandLocation]:
+        return eo_bands_to_band_locations(self.item, eo_bands)
+
+    def assets_eo_bands_to_band_locations(
+        self,
+        assets: Optional[List[str]] = None,
+        eo_bands: Optional[List[str]] = None,
+    ) -> List[BandLocation]:
+        assets = assets or []
+        eo_bands = eo_bands or []
+        if assets and eo_bands:
+            raise ValueError("assets and eo_bands cannot be provided at the same time")
+        if assets:
+            return [BandLocation(asset_name=asset) for asset in assets]
+        elif eo_bands:
+            return self.eo_bands_to_band_location(eo_bands)
+        else:
+            raise ValueError("assets or eo_bands have to be provided")
+
+
+def eo_bands_to_band_locations(
+    item: Item,
+    eo_bands: List[str],
+    role: Literal["data", "reflectance", "visual"] = "data",
+) -> List[BandLocation]:
+    """
+    Map EO band names to asset locations.
+
+    Args:
+        item: STAC Item.
+        eo_bands: List of common band names.
+        role: Functional role of the assets.
+
+    Returns:
+        List[BandLocation]: List of location objects.
+    """
+    return [find_eo_band(item, eo_band, role=role) for eo_band in eo_bands]
+
+
+def find_eo_band(
+    item: Item,
+    eo_band_name: str,
+    role: Literal["data", "reflectance", "visual"] = "data",
+) -> BandLocation:
+    """
+    Tries to find the location of the most appropriate band using the EO band name.
+
+    This function looks into all assets and all eo bands for the given name and role.
+    """
+    results = []
+    for asset_name, asset in item.assets.items():
+        # search in eo:bands and alternatively in bands for eo:common_name
+        for band_index, band_info in enumerate(
+            asset.extra_fields.get("eo:bands", asset.extra_fields.get("bands", [])), 1
+        ):
+            if (
+                # if name matches eo band name
+                (
+                    eo_band_name == band_info.get("name")
+                    or eo_band_name == band_info.get("eo:common_name")
+                )
+                # if role is given, make sure it matches with desired role
+                and (asset.roles is None or role in asset.roles)
+            ):
+                results.append(
+                    BandLocation.from_asset(
+                        name=asset_name,
+                        band_index=band_index,
+                        asset=asset,
+                    )
+                )
+
+    if len(results) == 0:
+        raise KeyError(f"EO band {eo_band_name} not found in item assets")
+
+    elif len(results) == 1:
+        return results[0]
+
+    # if results are ambiguous, further filter them
+    else:
+        # only use locations which seem to have the original resolution
+        for matches in [_asset_name_equals_eo_name, _is_original_sampling]:
+            filtered_results = [
+                band_location for band_location in results if matches(band_location)
+            ]
+            if len(filtered_results) == 1:
+                return filtered_results[0]
+        else:  # pragma: no cover
+            raise ValueError(
+                f"EO band '{eo_band_name}' found in multiple assets: {', '.join(map(str, results))}"
+            )
+
+
+def _asset_name_equals_eo_name(band_location: BandLocation) -> bool:
+    return band_location.asset_name == band_location.eo_band_name
+
+
+def _is_original_sampling(band_location: BandLocation) -> bool:
+    return band_location.roles == [] or "sampling:original" in band_location.roles
+
+
+def add_to_blacklist(path: MPathLike, blacklist: Optional[MPath] = None) -> None:
+    blacklist = blacklist or mapchete_eo_settings.blacklist
+
+    if blacklist is None:
+        return
+
+    blacklist = MPath.from_inp(blacklist)
+
+    path = MPath.from_inp(path)
+
+    # make sure paths stay unique
+    if str(path) not in blacklist_products(blacklist):
+        logger.debug("add path %s to blacklist", str(path))
+        try:
+            with blacklist.open("a") as dst:
+                dst.write(f"{path}\n")
+        except FileNotFoundError:
+            with blacklist.open("w") as dst:
+                dst.write(f"{path}\n")
+
+
+def blacklist_products(blacklist: Optional[MPathLike] = None) -> Set[str]:
+    blacklist = blacklist or mapchete_eo_settings.blacklist
+    if blacklist is None:
+        raise ValueError("no blacklist is defined")
+    blacklist = MPath.from_inp(blacklist)
+
+    try:
+        return set(blacklist.read_text().splitlines())
+    except FileNotFoundError:
+        logger.debug("%s does not exist, returning empty set", str(blacklist))
+        return set()

mapchete_eo/protocols.py

@@ -0,0 +1,61 @@
+from __future__ import annotations
+
+from typing import Any, Dict, List, Optional, Protocol
+
+import numpy.ma as ma
+import pystac
+import xarray as xr
+from mapchete.protocols import GridProtocol
+from mapchete.types import Bounds, NodataVals
+from rasterio.crs import CRS
+from rasterio.enums import Resampling
+
+from mapchete_eo.types import DateTimeLike
+from mapchete.io.raster import ReferencedRaster
+
+
+class EOProductProtocol(Protocol):
+    id: str
+    bounds: Bounds
+    crs: CRS
+    __geo_interface__: Optional[Dict[str, Any]]
+
+    @classmethod
+    def from_stac_item(self, item: pystac.Item, **kwargs) -> EOProductProtocol: ...
+
+    def get_mask(self) -> ReferencedRaster: ...
+
+    def read(
+        self,
+        assets: Optional[List[str]] = None,
+        eo_bands: Optional[List[str]] = None,
+        grid: Optional[GridProtocol] = None,
+        resampling: Resampling = Resampling.nearest,
+        nodatavals: NodataVals = None,
+        x_axis_name: str = "x",
+        y_axis_name: str = "y",
+        **kwargs,
+    ) -> xr.Dataset: ...
+
+    def read_np_array(
+        self,
+        assets: Optional[List[str]] = None,
+        eo_bands: Optional[List[str]] = None,
+        grid: Optional[GridProtocol] = None,
+        resampling: Resampling = Resampling.nearest,
+        nodatavals: NodataVals = None,
+        **kwargs,
+    ) -> ma.MaskedArray: ...
+
+    def get_property(self, property: str) -> Any: ...
+
+    @property
+    def item(self) -> pystac.Item: ...
+
+
+class DateTimeProtocol(Protocol):
+    datetime: DateTimeLike
+
+
+class GetPropertyProtocol(Protocol):
+    def get_property(self, property: str) -> Any: ...

mapchete_eo/search/__init__.py

@@ -0,0 +1,14 @@
+"""
+A catalog is an instance with a specific endpoint and a specific collection.
+
+The catalog class aims to abstract product search as well as homogenization
+of product metadata.
+
+It helps the InputData class to find the input products and their metadata.
+"""
+
+from mapchete_eo.search.stac_search import STACSearchCollection
+from mapchete_eo.search.stac_static import STACStaticCollection
+from mapchete_eo.search.utm_search import UTMSearchCatalog
+
+__all__ = ["STACSearchCollection", "STACStaticCollection", "UTMSearchCatalog"]

mapchete_eo/search/base.py

@@ -0,0 +1,285 @@
+from functools import cached_property
+import json
+import logging
+from abc import ABC, abstractmethod
+from typing import Any, Callable, Dict, Generator, List, Optional, Set, Type, Union
+
+from pygeofilter.parsers.ecql import parse as parse_ecql
+from pygeofilter.backends.native.evaluate import NativeEvaluator
+from pydantic import BaseModel
+from mapchete.path import MPath, MPathLike
+from mapchete.types import Bounds
+from pystac import Catalog, Item, CatalogType, Extent
+from pystac.collection import Collection
+from pystac.stac_io import DefaultStacIO
+from pystac_client import CollectionClient
+from pystac_client.stac_api_io import StacApiIO
+from rasterio.profiles import Profile
+from shapely.geometry.base import BaseGeometry
+
+from mapchete_eo.io.assets import get_assets, get_metadata_assets
+from mapchete_eo.product import blacklist_products
+from mapchete_eo.settings import mapchete_eo_settings
+from mapchete_eo.types import TimeRange
+
+logger = logging.getLogger(__name__)
+
+
+class FSSpecStacIO(StacApiIO):
+    """Custom class which allows I/O operations on object storage."""
+
+    def read_text(self, source: MPathLike, *args, **kwargs) -> str:
+        return MPath.from_inp(source).read_text()
+
+    def write_text(self, dest: MPathLike, txt: str, *args, **kwargs) -> None:
+        path = MPath.from_inp(dest)
+        if not path.parent.exists():
+            path.parent.makedirs(exist_ok=True)
+        with path.open("w") as dst:
+            return dst.write(txt)
+
+    # TODO: investigate in pystac why this has to be a staticmethod
+    @staticmethod
+    def save_json(dest: MPathLike, json_dict: dict, *args, **kwargs) -> None:
+        path = MPath.from_inp(dest)
+        if not path.parent.exists():
+            path.parent.makedirs(exist_ok=True)
+        with path.open("w") as dst:
+            return dst.write(json.dumps(json_dict, indent=2))
+
+
+class CollectionSearcher(ABC):
+    """
+    Bridge between a Source and a catalog implementation.
+    """
+
+    config_cls: Type[BaseModel]
+    collection: str
+    stac_item_modifiers: Optional[List[Callable[[Item], Item]]] = None
+    blacklist: Set[str] = (
+        blacklist_products(mapchete_eo_settings.blacklist)
+        if mapchete_eo_settings.blacklist
+        else set()
+    )
+
+    def __init__(
+        self,
+        collection: str,
+        stac_item_modifiers: Optional[List[Callable[[Item], Item]]] = None,
+    ):
+        self.collection = collection
+        self.stac_item_modifiers = stac_item_modifiers
+
+    @abstractmethod
+    @cached_property
+    def client(self) -> CollectionClient: ...
+
+    @abstractmethod
+    @cached_property
+    def eo_bands(self) -> List[str]: ...
+
+    @property
+    def config(self) -> BaseModel:
+        return self.config_cls()
+
+    @cached_property
+    def id(self) -> str:
+        return self.client.id
+
+    @cached_property
+    def description(self) -> str:
+        return self.client.description
+
+    @cached_property
+    def stac_extensions(self) -> List[str]:
+        return self.client.stac_extensions
+
+    @abstractmethod
+    def search(
+        self,
+        time: Optional[Union[TimeRange, List[TimeRange]]] = None,
+        bounds: Optional[Bounds] = None,
+        area: Optional[BaseGeometry] = None,
+        query: Optional[str] = None,
+        search_kwargs: Optional[Dict[str, Any]] = None,
+    ) -> Generator[Item, None, None]: ...
+
+
+class StaticCollectionWriterMixin(CollectionSearcher):
+    # client: Client
+    # id: str
+    # description: str
+    # stac_extensions: List[str]
+
+    def write_static_catalog(
+        self,
+        output_path: MPathLike,
+        bounds: Optional[Bounds] = None,
+        area: Optional[BaseGeometry] = None,
+        time: Optional[TimeRange] = None,
+        search_kwargs: Optional[Dict[str, Any]] = None,
+        name: Optional[str] = None,
+        description: Optional[str] = None,
+        assets: Optional[List[str]] = None,
+        assets_dst_resolution: Union[None, float, int] = None,
+        assets_convert_profile: Optional[Profile] = None,
+        copy_metadata: bool = False,
+        metadata_parser_classes: Optional[tuple] = None,
+        overwrite: bool = False,
+        stac_io: DefaultStacIO = FSSpecStacIO(),
+        progress_callback: Optional[Callable] = None,
+    ) -> MPath:
+        """
+        Export a static STAC catalog from the search results.
+
+        Args:
+            output_path: Destination directory for the static catalog.
+            bounds: Spatial filter bounds.
+            area: Spatial filter geometry.
+            time: Temporal filter range.
+            search_kwargs: Additional search arguments.
+            name: Catalog name.
+            description: Catalog description.
+            assets: List of assets to download.
+            assets_dst_resolution: Sub-sampling resolution for assets.
+            assets_convert_profile: Output profile for assets (e.g. for COG conversion).
+            copy_metadata: Whether to copy sidecar metadata files.
+            metadata_parser_classes: Custom parser classes for metadata.
+            overwrite: Overwrite existing files.
+            stac_io: Custom STAC IO implementation.
+            progress_callback: Optional function for progress reporting.
+
+        Returns:
+            MPath: Path to the generated catalog.json.
+        """
+        output_path = MPath.from_inp(output_path)
+        assets = assets or []
+        # initialize catalog
+        catalog_json = output_path / "catalog.json"
+        if catalog_json.exists():
+            logger.debug("open existing catalog %s", str(catalog_json))
+            catalog = Catalog.from_file(catalog_json)
+            # client = Client.from_file(catalog_json)
+            # existing_collection = client.get_collection(self.id)
+        else:
+            # existing_collections = []
+            catalog = Catalog(
+                name or f"{self.id}",
+                description or f"Static subset of {self.description}",
+                stac_extensions=self.stac_extensions,
+                href=str(catalog_json),
+                catalog_type=CatalogType.SELF_CONTAINED,
+            )
+        src_items = list(
+            self.search(
+                time=time, bounds=bounds, area=area, search_kwargs=search_kwargs
+            )
+        )
+        # collect all items and download assets if required
+        items: List[Item] = []
+        item_ids = set()
+        for n, item in enumerate(src_items, 1):
+            logger.debug("found item %s", item)
+            item = item.clone()
+            if assets:
+                logger.debug("get assets %s", assets)
+                item = get_assets(
+                    item,
+                    assets,
+                    output_path / self.id / item.id,
+                    resolution=assets_dst_resolution,
+                    convert_profile=assets_convert_profile,
+                    overwrite=overwrite,
+                    ignore_if_exists=True,
+                )
+            if copy_metadata:
+                item = get_metadata_assets(
+                    item,
+                    output_path / self.id / item.id,
+                    metadata_parser_classes=metadata_parser_classes,
+                    resolution=assets_dst_resolution,
+                    convert_profile=assets_convert_profile,
+                    overwrite=overwrite,
+                )
+            # this has to be set to None, otherwise pystac will mess up the asset paths
+            # after normalizing
+            item.set_self_href(None)
+
+            items.append(item)
+            item_ids.add(item.id)
+
+            if progress_callback:
+                progress_callback(n=n, total=len(src_items))
+
+            # for existing_collection in existing_collections:
+            #     if existing_collection.id == collection.id:
+            #         logger.debug("try to find unregistered items in collection")
+            #         collection_root_path = MPath.from_inp(
+            #             existing_collection.get_self_href()
+            #         ).parent
+            #         for subpath in collection_root_path.ls():
+            #             if subpath.is_directory():
+            #                 try:
+            #                     item = Item.from_file(
+            #                         subpath / subpath.with_suffix(".json").name
+            #                     )
+            #                     if item.id not in item_ids:
+            #                         logger.debug(
+            #                             "add existing item with id %s", item.id
+            #                         )
+            #                         items.append(item)
+            #                         item_ids.add(item.id)
+            #                 except FileNotFoundError:
+            #                     pass
+            #         break
+            # create collection and copy metadata
+            logger.debug("create new collection")
+            out_collection = Collection(
+                id=self.id,
+                extent=Extent.from_items(items),
+                description=self.description,
+                title=self.client.title,
+                stac_extensions=self.stac_extensions,
+                license=self.client.license,
+                keywords=self.client.keywords,
+                providers=self.client.providers,
+                summaries=self.client.summaries,
+                extra_fields=self.client.extra_fields,
+                catalog_type=CatalogType.SELF_CONTAINED,
+            )
+
+            # finally, add all items to collection
+            for item in items:
+                out_collection.add_item(item)
+
+            out_collection.update_extent_from_items()
+
+            catalog.add_child(out_collection)
+
+        logger.debug("write catalog to %s", output_path)
+        catalog.normalize_hrefs(str(output_path))
+        catalog.make_all_asset_hrefs_relative()
+        catalog.save(dest_href=str(output_path), stac_io=stac_io)
+
+        return catalog_json
+
+
+def filter_items(
+    items: Generator[Item, None, None],
+    query: Optional[str] = None,
+) -> Generator[Item, None, None]:
+    """
+    Only for cloudcover now, this can and should be adapted for filter field and value
+    the field and value for the item filter would be defined in search.config.py corresponding configs
+    and passed down to the individual search approaches via said config and this Function.
+    """
+    if query:
+        ast = parse_ecql(query)
+        evaluator = NativeEvaluator(use_getattr=False)
+        filter_func = evaluator.evaluate(ast)
+        for item in items:
+            # pystac items store metadata in 'properties'
+            if filter_func(item.properties):
+                yield item
+    else:
+        yield from items