mapchete-eo 2026.2.0__py3-none-any.whl

mapchete_eo/search/stac_static.py

@@ -0,0 +1,197 @@
+from functools import cached_property
+import logging
+import warnings
+from typing import Any, Dict, Generator, List, Optional, Union
+
+from mapchete import Bounds
+from mapchete.types import BoundsLike
+from pystac import Item, Catalog, Collection
+from mapchete.io.vector import bounds_intersect
+from pystac.stac_io import StacIO
+from pystac_client import CollectionClient
+from shapely.geometry import shape
+from shapely.geometry.base import BaseGeometry
+
+from mapchete_eo.search.base import (
+    CollectionSearcher,
+    FSSpecStacIO,
+    StaticCollectionWriterMixin,
+    filter_items,
+)
+from mapchete_eo.search.config import StacStaticConfig
+from mapchete_eo.time import time_ranges_intersect
+from mapchete_eo.types import TimeRange
+
+logger = logging.getLogger(__name__)
+
+
+StacIO.set_default(FSSpecStacIO)
+
+
+class STACStaticCollection(StaticCollectionWriterMixin, CollectionSearcher):
+    """
+    Search implementation for static STAC collections.
+    """
+
+    config_cls = StacStaticConfig
+
+    @cached_property
+    def client(self) -> CollectionClient:
+        return CollectionClient.from_file(str(self.collection), stac_io=FSSpecStacIO())
+
+    @cached_property
+    def eo_bands(self) -> List[str]:
+        eo_bands = self.client.extra_fields.get("properties", {}).get("eo:bands")
+        if eo_bands:
+            return eo_bands
+        else:
+            warnings.warn(
+                "Unable to read eo:bands definition from collection. "
+                "Trying now to get information from assets ..."
+            )
+            # see if eo:bands can be found in properties
+            try:
+                item = next(self.client.get_items(recursive=True))
+                eo_bands = item.properties.get("eo:bands")
+                if eo_bands:
+                    return eo_bands
+
+                # look through the assets and collect eo:bands
+                out = {}
+                for asset in item.assets.values():
+                    for eo_band in asset.extra_fields.get("eo:bands", []):
+                        out[eo_band["name"]] = eo_band
+                if out:
+                    return [v for v in out.values()]
+            except StopIteration:
+                pass
+
+            logger.debug("cannot find eo:bands definition")
+            return []
+
+    def search(
+        self,
+        time: Optional[Union[TimeRange, List[TimeRange]]] = None,
+        bounds: Optional[BoundsLike] = None,
+        area: Optional[BaseGeometry] = None,
+        query: Optional[str] = None,
+        search_kwargs: Optional[Dict[str, Any]] = None,
+    ) -> Generator[Item, None, None]:
+        if area is None and bounds:
+            bounds = Bounds.from_inp(bounds)
+            area = shape(bounds)
+        for item in filter_items(self._raw_search(time=time, area=area), query=query):
+            yield item
+
+    def _raw_search(
+        self,
+        time: Optional[Union[TimeRange, List[TimeRange]]] = None,
+        area: Optional[BaseGeometry] = None,
+    ) -> Generator[Item, None, None]:
+        if area is not None and area.is_empty:
+            return
+        logger.debug("iterate through children")
+        if time:
+            for time_range in time if isinstance(time, list) else [time]:
+                for item in _all_intersecting_items(
+                    self.client,
+                    area=area,
+                    time_range=time_range,
+                ):
+                    item.make_asset_hrefs_absolute()
+                    yield item
+        else:
+            for item in _all_intersecting_items(
+                self.client,
+                area=area,
+            ):
+                item.make_asset_hrefs_absolute()
+                yield item
+
+
+def _all_intersecting_items(
+    collection: Union[Catalog, Collection],
+    area: BaseGeometry,
+    time_range: Optional[TimeRange] = None,
+):
+    # collection items
+    logger.debug("checking items...")
+    for item in collection.get_items():
+        # yield item if it intersects with extent
+        logger.debug("item %s", item.id)
+        if _item_extent_intersects(item, area=area, time_range=time_range):
+            logger.debug("item %s within search parameters", item.id)
+            yield item
+
+    # collection children
+    logger.debug("checking collections...")
+    for child in collection.get_children():
+        # yield collection if it intersects with extent
+        logger.debug("collection %s", collection.id)
+        if _collection_extent_intersects(child, area=area, time_range=time_range):
+            logger.debug("found catalog %s with intersecting items", child.id)
+            yield from _all_intersecting_items(child, area=area, time_range=time_range)
+
+
+def _item_extent_intersects(
+    item: Item,
+    area: Optional[BaseGeometry] = None,
+    time_range: Optional[TimeRange] = None,
+) -> bool:
+    # NOTE: bounds intersect is faster but in the current implementation cannot
+    # handle item footprints going over the Antimeridian (and have been split up into
+    # MultiPolygon geometries)
+    # spatial_intersect = bounds_intersect(item.bbox, bounds) if bounds else True
+    spatial_intersect = shape(item.geometry).intersects(area) if area else True
+    if time_range and item.datetime:
+        temporal_intersect = time_ranges_intersect(
+            (item.datetime, item.datetime), (time_range.start, time_range.end)
+        )
+        logger.debug(
+            "spatial intersect: %s, temporal intersect: %s",
+            spatial_intersect,
+            temporal_intersect,
+        )
+        return spatial_intersect and temporal_intersect
+    else:
+        logger.debug("spatial intersect: %s", spatial_intersect)
+        return spatial_intersect
+
+
+def _collection_extent_intersects(
+    catalog, area: Optional[BaseGeometry] = None, time_range: Optional[TimeRange] = None
+):
+    """
+    Collection extent items (spatial, temporal) is a list of items, e.g. list of bounds values.
+    """
+
+    def _intersects_spatially():
+        for b in catalog.extent.spatial.to_dict().get("bbox", [[]]):
+            if bounds_intersect(area.bounds, b):
+                logger.debug("spatial intersect: True")
+                return True
+        else:
+            logger.debug("spatial intersect: False")
+            return False
+
+    def _intersects_temporally():
+        for t in catalog.extent.temporal.to_dict().get("interval", [[]]):
+            if time_ranges_intersect((time_range.start, time_range.end), t):
+                logger.debug("temporal intersect: True")
+                return True
+        else:
+            logger.debug("temporal intersect: False")
+            return False
+
+    spatial_intersect = _intersects_spatially() if area else True
+    if time_range:
+        temporal_intersect = _intersects_temporally()
+        logger.debug(
+            "spatial intersect: %s, temporal intersect: %s",
+            spatial_intersect,
+            temporal_intersect,
+        )
+        return spatial_intersect and temporal_intersect
+    else:
+        logger.debug("spatial intersect: %s", spatial_intersect)
+        return spatial_intersect

mapchete_eo/search/utm_search.py

@@ -0,0 +1,251 @@
+import datetime
+from functools import cached_property
+import logging
+from typing import Any, Dict, Generator, List, Optional, Union
+
+from mapchete.io.vector import fiona_open
+from mapchete.path import MPath, MPathLike
+from mapchete.types import Bounds, BoundsLike
+from pystac.collection import Collection
+from pystac.item import Item
+from pystac_client import CollectionClient
+from shapely.errors import GEOSException
+from shapely.geometry import shape
+from shapely.geometry.base import BaseGeometry
+
+from mapchete_eo.exceptions import ItemGeometryError
+from mapchete_eo.search.base import (
+    CollectionSearcher,
+    StaticCollectionWriterMixin,
+    filter_items,
+)
+from mapchete_eo.search.config import UTMSearchConfig
+from mapchete_eo.search.s2_mgrs import S2Tile, s2_tiles_from_bounds
+from mapchete_eo.time import day_range, to_datetime
+from mapchete_eo.types import TimeRange
+
+logger = logging.getLogger(__name__)
+
+
+class UTMSearchCatalog(StaticCollectionWriterMixin, CollectionSearcher):
+    """
+    Search implementation for UTM-grid based catalogs (e.g. Sentinel-2 on AWS).
+    """
+
+    config_cls = UTMSearchConfig
+
+    @cached_property
+    def endpoint(self) -> Optional[str]:
+        for collection_properties in self.config.sinergise_aws_collections.values():
+            if collection_properties["id"] == self.collection.split("/")[-1].replace(
+                ".json", ""
+            ):
+                return collection_properties.get("endpoint")
+        return None
+
+    day_subdir_schema: str = "{year}/{month:02d}/{day:02d}"
+    stac_json_endswith: str = "T{tile_id}.json"
+
+    @cached_property
+    def client(self) -> CollectionClient:
+        return next(self.get_collections())
+
+    @cached_property
+    def eo_bands(self) -> List[str]:  # pragma: no cover
+        for (
+            collection_properties
+        ) in UTMSearchConfig().sinergise_aws_collections.values():
+            if collection_properties["id"] == self.collection.split("/")[-1]:
+                collection = Collection.from_dict(
+                    collection_properties["path"].read_json()
+                )
+                if collection:
+                    summary = collection.summaries.to_dict()
+                    if "eo:bands" in summary:
+                        return summary["eo:bands"]
+                else:
+                    raise ValueError(f"cannot find collection {collection}")
+        else:
+            logger.debug(
+                "cannot find eo:bands definition from collection %s",
+                self.collection,
+            )
+            return []
+
+    def search(
+        self,
+        time: Optional[Union[TimeRange, List[TimeRange]]] = None,
+        bounds: Optional[BoundsLike] = None,
+        area: Optional[BaseGeometry] = None,
+        query: Optional[str] = None,
+        search_kwargs: Optional[Dict[str, Any]] = None,
+    ) -> Generator[Item, None, None]:
+        for item in filter_items(
+            self._raw_search(
+                time=time, bounds=Bounds.from_inp(bounds) if bounds else None, area=area
+            ),
+            query=query,
+        ):
+            yield item
+
+    def _raw_search(
+        self,
+        time: Optional[Union[TimeRange, List[TimeRange]]] = None,
+        bounds: Optional[Bounds] = None,
+        area: Optional[BaseGeometry] = None,
+        config: Optional[UTMSearchConfig] = None,
+    ) -> Generator[Item, None, None]:
+        config = config or UTMSearchConfig()
+        if time is None:
+            raise ValueError("time must be given")
+        if area is not None and area.is_empty:
+            return
+        if area is not None:
+            area = area
+            bounds = Bounds.from_inp(area)
+        elif bounds is not None:
+            bounds = Bounds.from_inp(bounds)
+            area = shape(bounds)
+
+        # Cleaner time list in case None present as time (undefined)
+        time_list: list[TimeRange] = (
+            [t for t in time if t is not None] if isinstance(time, list) else [time]
+        )
+        for time_range in time_list:
+            start_time = (
+                time_range.start
+                if isinstance(time_range.start, datetime.date)
+                else datetime.datetime.strptime(time_range.start, "%Y-%m-%d")
+            )
+            end_time = (
+                time_range.end
+                if isinstance(time_range.end, datetime.date)
+                else datetime.datetime.strptime(time_range.end, "%Y-%m-%d")
+            )
+
+            logger.debug(
+                "determine items from %s to %s over %s...",
+                start_time,
+                end_time,
+                bounds,
+            )
+            if config.search_index:
+                logger.debug(
+                    "use existing search index at %s", str(config.search_index)
+                )
+                for item in items_from_static_index(
+                    bounds=bounds,
+                    start_time=start_time,
+                    end_time=end_time,
+                    index_path=config.search_index,
+                ):
+                    try:
+                        item_path = item.get_self_href()
+                        if item_path in self.blacklist:  # pragma: no cover
+                            logger.debug(
+                                "item %s found in blacklist and skipping", item_path
+                            )
+                        elif area.intersects(shape(item.geometry)):
+                            yield item
+                    except GEOSException as exc:
+                        raise ItemGeometryError(
+                            f"item {item.get_self_href()} geometry could not be resolved: {str(exc)}"
+                        )
+
+            else:
+                logger.debug("using dumb ls directory search at %s", str(self.endpoint))
+                for item in items_from_directories(
+                    bounds=bounds,
+                    start_time=start_time,
+                    end_time=end_time,
+                    endpoint=self.endpoint,
+                    day_subdir_schema=self.day_subdir_schema,
+                    stac_json_endswith=self.stac_json_endswith,
+                ):
+                    item_path = item.get_self_href()
+                    if item_path in self.blacklist:  # pragma: no cover
+                        logger.debug(
+                            "item %s found in blacklist and skipping", item_path
+                        )
+                    elif area.intersects(shape(item.geometry)):
+                        yield item
+
+    def get_collections(self):
+        """
+        yeild transformed collection from:
+            https://sentinel-s2-l2a-stac.s3.amazonaws.com/sentinel-s2-l2a.json,
+            or https://sentinel-s2-l1c-stac.s3.amazonaws.com/sentinel-s2-l1c.json,
+            etc.
+        """
+        for collection_properties in self.config.sinergise_aws_collections.values():
+            collection = Collection.from_dict(collection_properties["path"].read_json())
+            if self.collection.split("/")[-1] == collection.id:
+                yield collection
+
+
+def items_from_static_index(
+    bounds: Bounds,
+    start_time: Union[datetime.datetime, datetime.date],
+    end_time: Union[datetime.datetime, datetime.date],
+    index_path: MPathLike,
+) -> Generator[Item, None, None]:
+    index_path = MPath.from_inp(index_path)
+
+    start_time = to_datetime(start_time)
+    # add day at end_time to include last day
+    end_time = to_datetime(end_time + datetime.timedelta(days=1))
+
+    # open index and determine which S2Tiles are covered
+    with fiona_open(index_path) as index:
+        # look at entries in every S2Tile and match with timestamp
+        for s2tile_feature in index.filter(bbox=bounds):
+            with fiona_open(
+                index_path.parent / s2tile_feature.properties["path"]
+            ) as s2tile:
+                for item_feature in s2tile.filter(bbox=bounds):
+                    # remove timezone info in order to compare with start_time and end_time
+                    timestamp = to_datetime(
+                        item_feature.properties["datetime"]
+                    ).replace(tzinfo=None)
+
+                    if start_time <= timestamp <= end_time:
+                        yield Item.from_dict(
+                            MPath.from_inp(item_feature.properties["path"]).read_json()
+                        )
+
+
+def items_from_directories(
+    bounds: Bounds,
+    start_time: Union[datetime.datetime, datetime.date],
+    end_time: Union[datetime.datetime, datetime.date],
+    endpoint: MPathLike,
+    day_subdir_schema: str = "{year}/{month:02d}/{day:02d}",
+    stac_json_endswith: str = "T{tile_id}.json",
+) -> Generator[Item, None, None]:
+    # get Sentinel-2 tiles over given bounds
+    s2_tiles = s2_tiles_from_bounds(*bounds)
+
+    # for each day within time range, look for tiles
+    for day in day_range(start_date=start_time, end_date=end_time):
+        day_path = MPath.from_inp(endpoint) / day_subdir_schema.format(
+            year=day.year, month=day.month, day=day.day
+        )
+        for item in find_items(
+            day_path,
+            s2_tiles,
+            product_endswith=stac_json_endswith,
+        ):
+            yield item
+
+
+def find_items(
+    path: MPath,
+    s2_tiles: List[S2Tile],
+    product_endswith: str = "T{tile_id}.json",
+) -> Generator[Item, None, None]:
+    match_parts = tuple(
+        product_endswith.format(tile_id=s2_tile.tile_id) for s2_tile in s2_tiles
+    )
+    for product_path in path.ls():
+        if product_path.endswith(match_parts):
+            yield Item.from_file(product_path)

mapchete_eo/settings.py

@@ -0,0 +1,25 @@
+from typing import Optional
+
+from mapchete.path import MPath, MPathLike
+from pydantic_settings import BaseSettings, SettingsConfigDict
+from rasterio.crs import CRS
+
+
+class Settings(BaseSettings):
+    """
+    Combine default settings with env variables.
+
+    All settings can be set in the environment by adding the 'MHUB_' prefix
+    and the settings in uppercase, e.g. MAPCHETE_EO_.
+    """
+
+    default_cache_location: MPathLike = MPath("s3://eox-mhub-cache/")
+    default_catalog_crs: CRS = CRS.from_epsg(4326)
+    blacklist: Optional[MPathLike] = None
+    lazy_load_stac_items: bool = True
+
+    # read from environment
+    model_config = SettingsConfigDict(env_prefix="MAPCHETE_EO_")
+
+
+mapchete_eo_settings: Settings = Settings()

mapchete_eo/sort.py

@@ -0,0 +1,60 @@
+"""
+This module holds all code required to sort products or slices.
+"""
+
+from typing import Callable, List, Optional
+
+from pydantic import BaseModel
+
+from mapchete_eo.protocols import DateTimeProtocol, GetPropertyProtocol
+from mapchete_eo.time import timedelta, to_datetime
+from mapchete_eo.types import DateTimeLike
+
+
+class SortMethodConfig(BaseModel):
+    func: Callable
+
+
+def sort_objects_by_target_date(
+    objects: List[DateTimeProtocol],
+    target_date: Optional[DateTimeLike] = None,
+    reverse: bool = False,
+    **kwargs,
+) -> List[DateTimeProtocol]:
+    """
+    Sort objects by their distance to a target date.
+    """
+    if len(objects) == 0:
+        return objects
+
+    if target_date is None:
+        time_list = [to_datetime(object.datetime) for object in objects]
+        start_time = min(time_list)
+        end_time = max(time_list)
+        target_datetime = start_time + (end_time - start_time) / 2
+    else:
+        target_datetime = to_datetime(target_date)
+
+    objects.sort(key=lambda x: timedelta(x.datetime, target_datetime), reverse=reverse)
+
+    return objects
+
+
+class TargetDateSort(SortMethodConfig):
+    func: Callable = sort_objects_by_target_date
+    target_date: Optional[DateTimeLike] = None
+    reverse: bool = False
+
+
+def sort_objects_by_cloud_cover(
+    objects: List[GetPropertyProtocol], reverse: bool = False
+) -> List[GetPropertyProtocol]:
+    if len(objects) == 0:  # pragma: no cover
+        return objects
+    objects.sort(key=lambda x: x.get_property("eo:cloud_cover"), reverse=reverse)
+    return objects
+
+
+class CloudCoverSort(SortMethodConfig):
+    func: Callable = sort_objects_by_cloud_cover
+    reverse: bool = False

mapchete_eo/source.py

@@ -0,0 +1,109 @@
+from functools import cached_property
+from typing import Any, Dict, List, Literal, Optional, Generator, Union, Callable
+
+from mapchete.bounds import Bounds
+from mapchete.path import MPath
+from mapchete.types import BoundsLike, CRSLike, MPathLike
+from pydantic import BaseModel, ConfigDict, model_validator
+from pystac import Item
+from shapely.geometry.base import BaseGeometry
+from shapely.errors import GEOSException
+
+from mapchete_eo.exceptions import ItemGeometryError
+from mapchete_eo.search.base import CollectionSearcher
+from mapchete_eo.search import STACSearchCollection, STACStaticCollection
+from mapchete_eo.settings import mapchete_eo_settings
+from mapchete_eo.types import TimeRange
+
+
+class Source(BaseModel):
+    """
+    All information required to consume EO products.
+    """
+
+    collection: str
+    catalog_crs: Optional[CRSLike] = mapchete_eo_settings.default_catalog_crs
+    query: Optional[str] = None
+    area: Optional[Union[MPathLike, dict, type[BaseGeometry]]] = None
+    bounds: Optional[BoundsLike] = None
+
+    model_config = ConfigDict(arbitrary_types_allowed=True)
+
+    @property
+    def item_modifier_funcs(self) -> List[Callable]:
+        return []
+
+    @cached_property
+    def catalog_type(self) -> Literal["search", "static"]:
+        # TODO: stupid test but probably sufficient
+        return "static" if self.collection.endswith(".json") else "search"
+
+    def _spatial_subset(
+        self,
+        bounds: Optional[BoundsLike] = None,
+        area: Optional[BaseGeometry] = None,
+    ) -> Dict[str, Any]:
+        """Combine bounds and area with bounds defined in Source if any."""
+        if self.bounds is None:
+            return {"bounds": bounds, "area": area}
+        self_bounds = Bounds.from_inp(self.bounds)
+        out = dict()
+        if bounds is not None:
+            bounds = Bounds.from_inp(bounds)
+            if bounds.intersects(self_bounds):
+                out["bounds"] = Bounds.from_inp(
+                    bounds.geometry.intersection(self_bounds.geometry)
+                )
+        if area is not None:
+            out["area"] = area.intersection(self_bounds.geometry)
+        return out
+
+    def search(
+        self,
+        time: Optional[Union[TimeRange, List[TimeRange]]] = None,
+        bounds: Optional[BoundsLike] = None,
+        area: Optional[BaseGeometry] = None,
+        base_dir: Optional[MPathLike] = None,
+    ) -> Generator[Item, None, None]:
+        for item in self.get_catalog(base_dir=base_dir).search(
+            time=time,
+            query=self.query,
+            search_kwargs=dict(query=self.query) if self.query else None,
+            **self._spatial_subset(
+                bounds=bounds,
+                area=area,
+            ),
+        ):
+            yield self.apply_item_modifier_funcs(item)
+
+    def apply_item_modifier_funcs(self, item: Item) -> Item:
+        try:
+            for modifier in self.item_modifier_funcs:
+                item = modifier(item)
+        except GEOSException as exc:
+            raise ItemGeometryError(
+                f"item {item.get_self_href()} geometry could not be resolved: {str(exc)}"
+            )
+        return item
+
+    def get_catalog(self, base_dir: Optional[MPathLike] = None) -> CollectionSearcher:
+        match self.catalog_type:
+            case "search":
+                return STACSearchCollection(self.collection)
+            case "static":
+                return STACStaticCollection(
+                    collection=MPath(self.collection).absolute_path(base_dir=base_dir)
+                )
+
+    def eo_bands(self, base_dir: Optional[MPathLike] = None) -> List[str]:
+        return self.get_catalog(base_dir=base_dir).eo_bands
+
+    @model_validator(mode="before")
+    def deprecate_max_cloud_cover(cls, values: Dict[str, Any]) -> Dict[str, Any]:
+        if "max_cloud_cover" in values:
+            raise DeprecationWarning(
+                "'max_cloud_cover' will be deprecated soon. Please use 'eo:cloud_cover<=...' in the source 'query' field.",
+            )
+        elif "area" in values:  # pragma: no cover
+            raise NotImplementedError("please use 'bounds' as spatial subset for now")
+        return values