mapchete-eo 2025.7.0__py2.py3-none-any.whl

mapchete_eo/search/stac_static.py

@@ -0,0 +1,236 @@
+import logging
+import warnings
+from typing import Any, Callable, Dict, Generator, List, Optional, Union
+
+from mapchete import Bounds
+from mapchete.types import BoundsLike
+from pystac import Item, Catalog, Collection
+from mapchete.io.vector import bounds_intersect
+from mapchete.path import MPathLike
+from pystac.stac_io import StacIO
+from pystac_client import Client
+from shapely.geometry import shape
+from shapely.geometry.base import BaseGeometry
+
+from mapchete_eo.search.base import (
+    CatalogSearcher,
+    FSSpecStacIO,
+    StaticCatalogWriterMixin,
+    filter_items,
+)
+from mapchete_eo.search.config import StacStaticConfig
+from mapchete_eo.time import time_ranges_intersect
+from mapchete_eo.types import TimeRange
+
+logger = logging.getLogger(__name__)
+
+
+StacIO.set_default(FSSpecStacIO)
+
+
+class STACStaticCatalog(StaticCatalogWriterMixin, CatalogSearcher):
+    config_cls = StacStaticConfig
+
+    def __init__(
+        self,
+        baseurl: MPathLike,
+        stac_item_modifiers: Optional[List[Callable[[Item], Item]]] = None,
+    ):
+        self.client = Client.from_file(str(baseurl), stac_io=FSSpecStacIO())
+        self.id = self.client.id
+        self.description = self.client.description
+        self.stac_extensions = self.client.stac_extensions
+        self.collections = [c.id for c in self.client.get_children()]
+        self.eo_bands = self._eo_bands()
+        self.stac_item_modifiers = stac_item_modifiers
+
+    def search(
+        self,
+        time: Optional[Union[TimeRange, List[TimeRange]]] = None,
+        bounds: Optional[BoundsLike] = None,
+        area: Optional[BaseGeometry] = None,
+        search_kwargs: Optional[Dict[str, Any]] = None,
+    ) -> Generator[Item, None, None]:
+        config = self.config_cls(**search_kwargs or {})
+        if area is None and bounds:
+            bounds = Bounds.from_inp(bounds)
+            area = shape(bounds)
+        for item in filter_items(
+            self._raw_search(time=time, area=area),
+            max_cloud_cover=config.max_cloud_cover,
+        ):
+            yield item
+
+    def _raw_search(
+        self,
+        time: Optional[Union[TimeRange, List[TimeRange]]] = None,
+        area: Optional[BaseGeometry] = None,
+    ) -> Generator[Item, None, None]:
+        if area is not None and area.is_empty:
+            return
+        logger.debug("iterate through children")
+        for collection in self.client.get_collections():
+            if time:
+                for time_range in time if isinstance(time, list) else [time]:
+                    for item in _all_intersecting_items(
+                        collection,
+                        area=area,
+                        time_range=time_range,
+                    ):
+                        item.make_asset_hrefs_absolute()
+                        yield item
+            else:
+                for item in _all_intersecting_items(
+                    collection,
+                    area=area,
+                ):
+                    item.make_asset_hrefs_absolute()
+                    yield item
+
+    def _eo_bands(self) -> List[str]:
+        for collection in self.client.get_children():
+            eo_bands = collection.extra_fields.get("properties", {}).get("eo:bands")
+            if eo_bands:
+                return eo_bands
+        else:
+            warnings.warn(
+                "Unable to read eo:bands definition from collections. "
+                "Trying now to get information from assets ..."
+            )
+
+            # see if eo:bands can be found in properties
+            item = _get_first_item(self.client.get_children())
+            eo_bands = item.properties.get("eo:bands")
+            if eo_bands:
+                return eo_bands
+
+            # look through the assets and collect eo:bands
+            out = {}
+            for asset in item.assets.values():
+                for eo_band in asset.extra_fields.get("eo:bands", []):
+                    out[eo_band["name"]] = eo_band
+            if out:
+                return [v for v in out.values()]
+
+            logger.debug("cannot find eo:bands definition")
+            return []
+
+    def get_collections(
+        self,
+        time: Optional[Union[TimeRange, List[TimeRange]]] = None,
+        bounds: Optional[BoundsLike] = None,
+        area: Optional[BaseGeometry] = None,
+    ):
+        if area is None and bounds is not None:
+            area = Bounds.from_inp(bounds).geometry
+        for collection in self.client.get_children():
+            if time:
+                for time_range in time if isinstance(time, list) else [time]:
+                    if _collection_extent_intersects(
+                        collection,
+                        area=area,
+                        time_range=time_range,
+                    ):
+                        yield collection
+            else:
+                if _collection_extent_intersects(collection, area=area):
+                    yield collection
+
+
+def _get_first_item(collections):
+    for collection in collections:
+        for item in collection.get_all_items():
+            return item
+        else:
+            for child in collection.get_children():
+                return _get_first_item(child)
+    else:
+        raise ValueError("collections contain no items")
+
+
+def _all_intersecting_items(
+    collection: Union[Catalog, Collection],
+    area: BaseGeometry,
+    time_range: Optional[TimeRange] = None,
+):
+    # collection items
+    logger.debug("checking items...")
+    for item in collection.get_items():
+        # yield item if it intersects with extent
+        logger.debug("item %s", item.id)
+        if _item_extent_intersects(item, area=area, time_range=time_range):
+            logger.debug("item %s within search parameters", item.id)
+            yield item
+
+    # collection children
+    logger.debug("checking collections...")
+    for child in collection.get_children():
+        # yield collection if it intersects with extent
+        logger.debug("collection %s", collection.id)
+        if _collection_extent_intersects(child, area=area, time_range=time_range):
+            logger.debug("found catalog %s with intersecting items", child.id)
+            yield from _all_intersecting_items(child, area=area, time_range=time_range)
+
+
+def _item_extent_intersects(
+    item: Item,
+    area: Optional[BaseGeometry] = None,
+    time_range: Optional[TimeRange] = None,
+) -> bool:
+    # NOTE: bounds intersect is faster but in the current implementation cannot
+    # handle item footprints going over the Antimeridian (and have been split up into
+    # MultiPolygon geometries)
+    # spatial_intersect = bounds_intersect(item.bbox, bounds) if bounds else True
+    spatial_intersect = shape(item.geometry).intersects(area) if area else True
+    if time_range and item.datetime:
+        temporal_intersect = time_ranges_intersect(
+            (item.datetime, item.datetime), (time_range.start, time_range.end)
+        )
+        logger.debug(
+            "spatial intersect: %s, temporal intersect: %s",
+            spatial_intersect,
+            temporal_intersect,
+        )
+        return spatial_intersect and temporal_intersect
+    else:
+        logger.debug("spatial intersect: %s", spatial_intersect)
+        return spatial_intersect
+
+
+def _collection_extent_intersects(
+    catalog, area: Optional[BaseGeometry] = None, time_range: Optional[TimeRange] = None
+):
+    """
+    Collection extent items (spatial, temporal) is a list of items, e.g. list of bounds values.
+    """
+
+    def _intersects_spatially():
+        for b in catalog.extent.spatial.to_dict().get("bbox", [[]]):
+            if bounds_intersect(area.bounds, b):
+                logger.debug("spatial intersect: True")
+                return True
+        else:
+            logger.debug("spatial intersect: False")
+            return False
+
+    def _intersects_temporally():
+        for t in catalog.extent.temporal.to_dict().get("interval", [[]]):
+            if time_ranges_intersect((time_range.start, time_range.end), t):
+                logger.debug("temporal intersect: True")
+                return True
+        else:
+            logger.debug("temporal intersect: False")
+            return False
+
+    spatial_intersect = _intersects_spatially() if area else True
+    if time_range:
+        temporal_intersect = _intersects_temporally()
+        logger.debug(
+            "spatial intersect: %s, temporal intersect: %s",
+            spatial_intersect,
+            temporal_intersect,
+        )
+        return spatial_intersect and temporal_intersect
+    else:
+        logger.debug("spatial intersect: %s", spatial_intersect)
+        return spatial_intersect

mapchete_eo/search/utm_search.py

@@ -0,0 +1,251 @@
+import datetime
+import logging
+from typing import Any, Callable, Dict, Generator, List, Optional, Set, Union
+
+from mapchete.io.vector import fiona_open
+from mapchete.path import MPath, MPathLike
+from mapchete.types import Bounds, BoundsLike
+from pystac.collection import Collection
+from pystac.item import Item
+from shapely.errors import GEOSException
+from shapely.geometry import shape
+from shapely.geometry.base import BaseGeometry
+
+from mapchete_eo.exceptions import ItemGeometryError
+from mapchete_eo.product import blacklist_products
+from mapchete_eo.search.base import (
+    CatalogSearcher,
+    StaticCatalogWriterMixin,
+    filter_items,
+)
+from mapchete_eo.search.config import UTMSearchConfig
+from mapchete_eo.search.s2_mgrs import S2Tile, s2_tiles_from_bounds
+from mapchete_eo.settings import mapchete_eo_settings
+from mapchete_eo.time import day_range, to_datetime
+from mapchete_eo.types import TimeRange
+
+logger = logging.getLogger(__name__)
+
+
+class UTMSearchCatalog(StaticCatalogWriterMixin, CatalogSearcher):
+    endpoint: str
+    id: str
+    day_subdir_schema: str
+    stac_json_endswith: str
+    description: str
+    stac_extensions: List[str]
+    blacklist: Set[str] = (
+        blacklist_products(mapchete_eo_settings.blacklist)
+        if mapchete_eo_settings.blacklist
+        else set()
+    )
+    config_cls = UTMSearchConfig
+
+    def __init__(
+        self,
+        endpoint: Optional[MPathLike] = None,
+        collections: List[str] = [],
+        stac_item_modifiers: Optional[List[Callable[[Item], Item]]] = None,
+    ):
+        self.endpoint = endpoint or self.endpoint
+        if len(collections) == 0:  # pragma: no cover
+            raise ValueError("no collections provided")
+        self.collections = collections
+        self.eo_bands = self._eo_bands()
+        self.stac_item_modifiers = stac_item_modifiers
+
+    def search(
+        self,
+        time: Optional[Union[TimeRange, List[TimeRange]]] = None,
+        bounds: Optional[BoundsLike] = None,
+        area: Optional[BaseGeometry] = None,
+        search_kwargs: Optional[Dict[str, Any]] = None,
+    ) -> Generator[Item, None, None]:
+        config = self.config_cls(**search_kwargs or {})
+        if bounds:
+            bounds = Bounds.from_inp(bounds)
+
+        for item in filter_items(
+            self._raw_search(time=time, bounds=bounds, area=area),
+            max_cloud_cover=config.max_cloud_cover,
+        ):
+            yield item
+
+    def _raw_search(
+        self,
+        time: Optional[Union[TimeRange, List[TimeRange]]] = None,
+        bounds: Optional[Bounds] = None,
+        area: Optional[BaseGeometry] = None,
+        config: UTMSearchConfig = UTMSearchConfig(),
+    ) -> Generator[Item, None, None]:
+        if time is None:
+            raise ValueError("time must be given")
+        if area is not None and area.is_empty:
+            return
+        if area is not None:
+            area = area
+            bounds = Bounds.from_inp(area)
+        elif bounds is not None:
+            bounds = Bounds.from_inp(bounds)
+            area = shape(bounds)
+        for time_range in time if isinstance(time, list) else [time]:
+            start_time = (
+                time_range.start
+                if isinstance(time_range.start, datetime.date)
+                else datetime.datetime.strptime(time_range.start, "%Y-%m-%d")
+            )
+            end_time = (
+                time_range.end
+                if isinstance(time_range.end, datetime.date)
+                else datetime.datetime.strptime(time_range.end, "%Y-%m-%d")
+            )
+
+            logger.debug(
+                "determine items from %s to %s over %s...",
+                start_time,
+                end_time,
+                bounds,
+            )
+            if config.search_index:
+                logger.debug(
+                    "use existing search index at %s", str(config.search_index)
+                )
+                for item in items_from_static_index(
+                    bounds=bounds,
+                    start_time=start_time,
+                    end_time=end_time,
+                    index_path=config.search_index,
+                ):
+                    try:
+                        item_path = item.get_self_href()
+                        if item_path in self.blacklist:  # pragma: no cover
+                            logger.debug(
+                                "item %s found in blacklist and skipping", item_path
+                            )
+                        elif area.intersects(shape(item.geometry)):
+                            yield item
+                    except GEOSException as exc:
+                        raise ItemGeometryError(
+                            f"item {item.get_self_href()} geometry could not be resolved: {str(exc)}"
+                        )
+
+            else:
+                logger.debug("using dumb ls directory search at %s", str(self.endpoint))
+                for item in items_from_directories(
+                    bounds=bounds,
+                    start_time=start_time,
+                    end_time=end_time,
+                    endpoint=self.endpoint,
+                    day_subdir_schema=self.day_subdir_schema,
+                    stac_json_endswith=self.stac_json_endswith,
+                ):
+                    item_path = item.get_self_href()
+                    if item_path in self.blacklist:  # pragma: no cover
+                        logger.debug(
+                            "item %s found in blacklist and skipping", item_path
+                        )
+                    elif area.intersects(shape(item.geometry)):
+                        yield item
+
+    def _eo_bands(self) -> list:
+        for collection_name in self.collections:
+            for (
+                collection_properties
+            ) in UTMSearchConfig().sinergise_aws_collections.values():
+                if collection_properties["id"] == collection_name:
+                    collection = Collection.from_dict(
+                        collection_properties["path"].read_json()
+                    )
+                    if collection:
+                        summary = collection.summaries.to_dict()
+                        if "eo:bands" in summary:
+                            return summary["eo:bands"]
+                    else:
+                        raise ValueError(f"cannot find collection {collection}")
+        else:
+            logger.debug(
+                "cannot find eo:bands definition from collections %s",
+                self.collections,
+            )
+            return []
+
+    def get_collections(self):
+        """
+        yeild transformed collection from:
+            https://sentinel-s2-l2a-stac.s3.amazonaws.com/sentinel-s2-l2a.json,
+            or https://sentinel-s2-l1c-stac.s3.amazonaws.com/sentinel-s2-l1c.json,
+            etc.
+        """
+        for collection_properties in self.config.sinergise_aws_collections.values():
+            collection = Collection.from_dict(collection_properties["path"].read_json())
+            for collection_name in self.collections:
+                if collection_name == collection.id:
+                    yield collection
+
+
+def items_from_static_index(
+    bounds: Bounds,
+    start_time: Union[datetime.datetime, datetime.date],
+    end_time: Union[datetime.datetime, datetime.date],
+    index_path: MPathLike,
+) -> Generator[Item, None, None]:
+    index_path = MPath.from_inp(index_path)
+
+    start_time = to_datetime(start_time)
+    # add day at end_time to include last day
+    end_time = to_datetime(end_time + datetime.timedelta(days=1))
+
+    # open index and determine which S2Tiles are covered
+    with fiona_open(index_path) as index:
+        # look at entries in every S2Tile and match with timestamp
+        for s2tile_feature in index.filter(bbox=bounds):
+            with fiona_open(
+                index_path.parent / s2tile_feature.properties["path"]
+            ) as s2tile:
+                for item_feature in s2tile.filter(bbox=bounds):
+                    # remove timezone info in order to compare with start_time and end_time
+                    timestamp = to_datetime(
+                        item_feature.properties["datetime"]
+                    ).replace(tzinfo=None)
+
+                    if start_time <= timestamp <= end_time:
+                        yield Item.from_dict(
+                            MPath.from_inp(item_feature.properties["path"]).read_json()
+                        )
+
+
+def items_from_directories(
+    bounds: Bounds,
+    start_time: Union[datetime.datetime, datetime.date],
+    end_time: Union[datetime.datetime, datetime.date],
+    endpoint: MPathLike,
+    day_subdir_schema: str = "{year}/{month:02d}/{day:02d}",
+    stac_json_endswith: str = "T{tile_id}.json",
+) -> Generator[Item, None, None]:
+    # get Sentinel-2 tiles over given bounds
+    s2_tiles = s2_tiles_from_bounds(*bounds)
+
+    # for each day within time range, look for tiles
+    for day in day_range(start_date=start_time, end_date=end_time):
+        day_path = MPath.from_inp(endpoint) / day_subdir_schema.format(
+            year=day.year, month=day.month, day=day.day
+        )
+        for item in find_items(
+            day_path,
+            s2_tiles,
+            product_endswith=stac_json_endswith,
+        ):
+            yield item
+
+
+def find_items(
+    path: MPath,
+    s2_tiles: List[S2Tile],
+    product_endswith: str = "T{tile_id}.json",
+) -> Generator[Item, None, None]:
+    match_parts = tuple(
+        product_endswith.format(tile_id=s2_tile.tile_id) for s2_tile in s2_tiles
+    )
+    for product_path in path.ls():
+        if product_path.endswith(match_parts):
+            yield Item.from_file(product_path)

mapchete_eo/settings.py

@@ -0,0 +1,24 @@
+from typing import Optional
+
+from mapchete.path import MPath, MPathLike
+from pydantic_settings import BaseSettings, SettingsConfigDict
+from rasterio.crs import CRS
+
+
+class Settings(BaseSettings):
+    """
+    Combine default settings with env variables.
+
+    All settings can be set in the environment by adding the 'MHUB_' prefix
+    and the settings in uppercase, e.g. MAPCHETE_EO_.
+    """
+
+    default_cache_location: MPathLike = MPath("s3://eox-mhub-cache/")
+    default_catalog_crs: CRS = CRS.from_epsg(4326)
+    blacklist: Optional[MPathLike] = None
+
+    # read from environment
+    model_config = SettingsConfigDict(env_prefix="MAPCHETE_EO_")
+
+
+mapchete_eo_settings: Settings = Settings()

mapchete_eo/sort.py

@@ -0,0 +1,48 @@
+"""
+This module holds all code required to sort products or slices.
+"""
+
+from typing import Callable, List, Optional
+
+from pydantic import BaseModel
+
+from mapchete_eo.protocols import DateTimeProtocol
+from mapchete_eo.time import timedelta, to_datetime
+from mapchete_eo.types import DateTimeLike
+
+
+class SortMethodConfig(BaseModel):
+    func: Callable
+
+
+def sort_objects_by_target_date(
+    objects: List[DateTimeProtocol],
+    target_date: Optional[DateTimeLike] = None,
+    reverse: bool = False,
+    **kwargs,
+) -> List[DateTimeProtocol]:
+    """
+    Return sorted list of onjects according to their distance to the target_date.
+
+    Default for target date is the middle between the objects start date and end date.
+    """
+    if len(objects) == 0:
+        return objects
+
+    if target_date is None:
+        time_list = [to_datetime(object.datetime) for object in objects]
+        start_time = min(time_list)
+        end_time = max(time_list)
+        target_datetime = start_time + (end_time - start_time) / 2
+    else:
+        target_datetime = to_datetime(target_date)
+
+    objects.sort(key=lambda x: timedelta(x.datetime, target_datetime), reverse=reverse)
+
+    return objects
+
+
+class TargetDateSort(SortMethodConfig):
+    func: Callable = sort_objects_by_target_date
+    target_date: Optional[DateTimeLike] = None
+    reverse: bool = False

mapchete_eo/time.py

@@ -0,0 +1,53 @@
+import datetime
+from typing import List, Tuple, Union
+
+import dateutil.parser
+
+from mapchete_eo.types import DateTimeLike
+
+_time = {"min": datetime.datetime.min.time(), "max": datetime.datetime.max.time()}
+
+
+def to_datetime(t: DateTimeLike, append_time="min") -> datetime.datetime:
+    """Convert input into datetime object."""
+    if isinstance(t, datetime.datetime):
+        return t
+    elif isinstance(t, datetime.date):
+        return datetime.datetime.combine(t, _time[append_time])
+    else:
+        return dateutil.parser.parse(t)
+
+
+def time_ranges_intersect(
+    t1: Tuple[DateTimeLike, DateTimeLike],
+    t2: Tuple[DateTimeLike, DateTimeLike],
+) -> bool:
+    """Check if two time ranges intersect."""
+    t1_start = to_datetime(t1[0], "min").replace(tzinfo=None)
+    t1_end = to_datetime(t1[1], "max").replace(tzinfo=None)
+    t2_start = to_datetime(t2[0], "min").replace(tzinfo=None)
+    t2_end = to_datetime(t2[1], "max").replace(tzinfo=None)
+    return (t1_start <= t2_start <= t1_end) or (t2_start <= t1_start <= t2_end)
+
+
+def timedelta(date: DateTimeLike, target: DateTimeLike, seconds: bool = True):
+    """Return difference between two time stamps."""
+    delta = to_datetime(date) - to_datetime(target)
+    if seconds:
+        return abs(delta.total_seconds())
+    else:
+        return abs(delta.days)
+
+
+def day_range(
+    start_date: Union[datetime.datetime, datetime.date],
+    end_date: Union[datetime.datetime, datetime.date],
+) -> List[datetime.date]:
+    start_date = (
+        start_date.date() if isinstance(start_date, datetime.datetime) else start_date
+    )
+    end_date = end_date.date() if isinstance(end_date, datetime.datetime) else end_date
+    return [
+        start_date + datetime.timedelta(n)
+        for n in range(int((end_date - start_date).days) + 1)
+    ]

mapchete_eo/types.py

@@ -0,0 +1,73 @@
+from __future__ import annotations
+
+import datetime
+from dataclasses import dataclass, field
+from enum import Enum
+from typing import List, Optional, Union
+
+from pydantic import PositiveInt
+from pystac import Asset
+
+
+class GeodataType(str, Enum):
+    vector = "vector"
+    raster = "raster"
+
+
+class MergeMethod(str, Enum):
+    """
+    Available methods to merge assets from multiple items.
+
+    first: first pixel value from the list is returned
+    average: average value from the list is returned
+    all: any consecutive value is added and all collected are returned
+    """
+
+    first = "first"
+    average = "average"
+    all = "all"
+
+
+DateLike = Union[str, datetime.date]
+DateTimeLike = Union[DateLike, datetime.datetime]
+
+
+@dataclass
+class BandLocation:
+    """A class representing the location of a specific band."""
+
+    asset_name: str
+    band_index: PositiveInt = 1
+    nodataval: float = 0
+    roles: List[str] = field(default_factory=list)
+    eo_band_name: Optional[str] = None
+
+    @staticmethod
+    def from_asset(
+        asset: Asset,
+        name: str,
+        band_index: PositiveInt,
+    ) -> BandLocation:
+        try:
+            bands_info = asset.extra_fields.get(
+                "eo:bands", asset.extra_fields.get("bands", [])
+            )
+            band_info = bands_info[band_index - 1]
+            eo_band_name = band_info.get("eo:common_name", band_info.get("name"))
+        except KeyError:
+            eo_band_name = None
+        return BandLocation(
+            asset_name=name,
+            band_index=band_index,
+            nodataval=asset.extra_fields.get("nodata", 0),
+            roles=asset.roles or [],
+            eo_band_name=eo_band_name,
+        )
+
+
+@dataclass
+class TimeRange:
+    """A class handling time ranges."""
+
+    start: DateTimeLike
+    end: DateTimeLike