mapchete-eo 2025.10.1__py2.py3-none-any.whl → 2025.11.0__py2.py3-none-any.whl

mapchete_eo/protocols.py

@@ -15,6 +15,7 @@ from mapchete.io.raster import ReferencedRaster
 
 
 class EOProductProtocol(Protocol):
+    id: str
     bounds: Bounds
     crs: CRS
     __geo_interface__: Optional[Dict[str, Any]]
@@ -54,3 +55,7 @@ class EOProductProtocol(Protocol):
 
 class DateTimeProtocol(Protocol):
     datetime: DateTimeLike
+
+
+class GetPropertyProtocol(Protocol):
+    def get_property(self, property: str) -> Any: ...

mapchete_eo/search/__init__.py

@@ -7,8 +7,8 @@ of product metadata.
 It helps the InputData class to find the input products and their metadata.
 """
 
-from mapchete_eo.search.stac_search import STACSearchCatalog
-from mapchete_eo.search.stac_static import STACStaticCatalog
+from mapchete_eo.search.stac_search import STACSearchCollection
+from mapchete_eo.search.stac_static import STACStaticCollection
 from mapchete_eo.search.utm_search import UTMSearchCatalog
 
-__all__ = ["STACSearchCatalog", "STACStaticCatalog", "UTMSearchCatalog"]
+__all__ = ["STACSearchCollection", "STACStaticCollection", "UTMSearchCatalog"]

mapchete_eo/search/base.py

@@ -4,13 +4,14 @@ import logging
 from abc import ABC, abstractmethod
 from typing import Any, Callable, Dict, Generator, List, Optional, Type, Union
 
+from cql2 import Expr
 from pydantic import BaseModel
-from pystac import Item, Catalog, CatalogType, Extent
 from mapchete.path import MPath, MPathLike
 from mapchete.types import Bounds
+from pystac import Catalog, Item, CatalogType, Extent
 from pystac.collection import Collection
 from pystac.stac_io import DefaultStacIO
-from pystac_client import Client
+from pystac_client import CollectionClient
 from pystac_client.stac_api_io import StacApiIO
 from rasterio.profiles import Profile
 from shapely.geometry.base import BaseGeometry
@@ -44,13 +45,26 @@ class FSSpecStacIO(StacApiIO):
             return dst.write(json.dumps(json_dict, indent=2))
 
 
-class CatalogSearcher(ABC):
+class CollectionSearcher(ABC):
     """
     This class serves as a bridge between an Archive and a catalog implementation.
     """
 
-    collections: List[str]
     config_cls: Type[BaseModel]
+    collection: str
+    stac_item_modifiers: Optional[List[Callable[[Item], Item]]] = None
+
+    def __init__(
+        self,
+        collection: str,
+        stac_item_modifiers: Optional[List[Callable[[Item], Item]]] = None,
+    ):
+        self.collection = collection
+        self.stac_item_modifiers = stac_item_modifiers
+
+    @abstractmethod
+    @cached_property
+    def client(self) -> CollectionClient: ...
 
     @abstractmethod
     @cached_property
@@ -74,20 +88,17 @@ class CatalogSearcher(ABC):
         time: Optional[Union[TimeRange, List[TimeRange]]] = None,
         bounds: Optional[Bounds] = None,
         area: Optional[BaseGeometry] = None,
+        query: Optional[str] = None,
         search_kwargs: Optional[Dict[str, Any]] = None,
     ) -> Generator[Item, None, None]: ...
 
 
-class StaticCatalogWriterMixin(CatalogSearcher):
+class StaticCollectionWriterMixin(CollectionSearcher):
     # client: Client
     # id: str
     # description: str
     # stac_extensions: List[str]
 
-    @abstractmethod
-    def get_collections(self) -> List[Collection]:  # pragma: no cover
-        ...
-
     def write_static_catalog(
         self,
         output_path: MPathLike,
@@ -113,94 +124,93 @@ class StaticCatalogWriterMixin(CatalogSearcher):
         catalog_json = output_path / "catalog.json"
         if catalog_json.exists():
             logger.debug("open existing catalog %s", str(catalog_json))
-            client = Client.from_file(catalog_json)
-            # catalog = pystac.Catalog.from_file(catalog_json)
-            existing_collections = list(client.get_collections())
+            catalog = Catalog.from_file(catalog_json)
+            # client = Client.from_file(catalog_json)
+            # existing_collection = client.get_collection(self.id)
         else:
-            existing_collections = []
-        catalog = Catalog(
-            name or f"{self.id}",
-            description or f"Static subset of {self.description}",
-            stac_extensions=self.stac_extensions,
-            href=str(catalog_json),
-            catalog_type=CatalogType.SELF_CONTAINED,
-        )
+            # existing_collections = []
+            catalog = Catalog(
+                name or f"{self.id}",
+                description or f"Static subset of {self.description}",
+                stac_extensions=self.stac_extensions,
+                href=str(catalog_json),
+                catalog_type=CatalogType.SELF_CONTAINED,
+            )
         src_items = list(
             self.search(
                 time=time, bounds=bounds, area=area, search_kwargs=search_kwargs
             )
         )
-        for collection in self.get_collections():
-            # collect all items and download assets if required
-            items: List[Item] = []
-            item_ids = set()
-            for n, item in enumerate(src_items, 1):
-                logger.debug("found item %s", item)
-                item = item.clone()
-                if assets:
-                    logger.debug("get assets %s", assets)
-                    item = get_assets(
-                        item,
-                        assets,
-                        output_path / collection.id / item.id,
-                        resolution=assets_dst_resolution,
-                        convert_profile=assets_convert_profile,
-                        overwrite=overwrite,
-                        ignore_if_exists=True,
-                    )
-                if copy_metadata:
-                    item = get_metadata_assets(
-                        item,
-                        output_path / collection.id / item.id,
-                        metadata_parser_classes=metadata_parser_classes,
-                        resolution=assets_dst_resolution,
-                        convert_profile=assets_convert_profile,
-                        overwrite=overwrite,
-                    )
-                # this has to be set to None, otherwise pystac will mess up the asset paths
-                # after normalizing
-                item.set_self_href(None)
-
-                items.append(item)
-                item_ids.add(item.id)
-
-                if progress_callback:
-                    progress_callback(n=n, total=len(src_items))
-
-            for existing_collection in existing_collections:
-                if existing_collection.id == collection.id:
-                    logger.debug("try to find unregistered items in collection")
-                    collection_root_path = MPath.from_inp(
-                        existing_collection.get_self_href()
-                    ).parent
-                    for subpath in collection_root_path.ls():
-                        if subpath.is_directory():
-                            try:
-                                item = Item.from_file(
-                                    subpath / subpath.with_suffix(".json").name
-                                )
-                                if item.id not in item_ids:
-                                    logger.debug(
-                                        "add existing item with id %s", item.id
-                                    )
-                                    items.append(item)
-                                    item_ids.add(item.id)
-                            except FileNotFoundError:
-                                pass
-                    break
+        # collect all items and download assets if required
+        items: List[Item] = []
+        item_ids = set()
+        for n, item in enumerate(src_items, 1):
+            logger.debug("found item %s", item)
+            item = item.clone()
+            if assets:
+                logger.debug("get assets %s", assets)
+                item = get_assets(
+                    item,
+                    assets,
+                    output_path / self.id / item.id,
+                    resolution=assets_dst_resolution,
+                    convert_profile=assets_convert_profile,
+                    overwrite=overwrite,
+                    ignore_if_exists=True,
+                )
+            if copy_metadata:
+                item = get_metadata_assets(
+                    item,
+                    output_path / self.id / item.id,
+                    metadata_parser_classes=metadata_parser_classes,
+                    resolution=assets_dst_resolution,
+                    convert_profile=assets_convert_profile,
+                    overwrite=overwrite,
+                )
+            # this has to be set to None, otherwise pystac will mess up the asset paths
+            # after normalizing
+            item.set_self_href(None)
+
+            items.append(item)
+            item_ids.add(item.id)
+
+            if progress_callback:
+                progress_callback(n=n, total=len(src_items))
+
+            # for existing_collection in existing_collections:
+            #     if existing_collection.id == collection.id:
+            #         logger.debug("try to find unregistered items in collection")
+            #         collection_root_path = MPath.from_inp(
+            #             existing_collection.get_self_href()
+            #         ).parent
+            #         for subpath in collection_root_path.ls():
+            #             if subpath.is_directory():
+            #                 try:
+            #                     item = Item.from_file(
+            #                         subpath / subpath.with_suffix(".json").name
+            #                     )
+            #                     if item.id not in item_ids:
+            #                         logger.debug(
+            #                             "add existing item with id %s", item.id
+            #                         )
+            #                         items.append(item)
+            #                         item_ids.add(item.id)
+            #                 except FileNotFoundError:
+            #                     pass
+            #         break
             # create collection and copy metadata
             logger.debug("create new collection")
             out_collection = Collection(
-                id=collection.id,
+                id=self.id,
                 extent=Extent.from_items(items),
-                description=collection.description,
-                title=collection.title,
-                stac_extensions=collection.stac_extensions,
-                license=collection.license,
-                keywords=collection.keywords,
-                providers=collection.providers,
-                summaries=collection.summaries,
-                extra_fields=collection.extra_fields,
+                description=self.description,
+                title=self.client.title,
+                stac_extensions=self.stac_extensions,
+                license=self.client.license,
+                keywords=self.client.keywords,
+                providers=self.client.providers,
+                summaries=self.client.summaries,
+                extra_fields=self.client.extra_fields,
                 catalog_type=CatalogType.SELF_CONTAINED,
             )
 
@@ -222,14 +232,17 @@ class StaticCatalogWriterMixin(CatalogSearcher):
 
 def filter_items(
     items: Generator[Item, None, None],
-    cloud_cover_field: str = "eo:cloud_cover",
-    max_cloud_cover: float = 100.0,
+    query: Optional[str] = None,
 ) -> Generator[Item, None, None]:
     """
     Only for cloudcover now, this can and should be adapted for filter field and value
     the field and value for the item filter would be defined in search.config.py corresponding configs
     and passed down to the individual search approaches via said config and this Function.
     """
-    for item in items:
-        if item.properties.get(cloud_cover_field, 0.0) <= max_cloud_cover:
-            yield item
+    if query:
+        expr = Expr(query)
+        for item in items:
+            if expr.matches(item.properties):
+                yield item
+    else:
+        yield from items

mapchete_eo/search/config.py

@@ -1,23 +1,44 @@
-from typing import Optional
+from typing import Optional, Dict, Any
 
 from mapchete.path import MPath, MPathLike
-from pydantic import BaseModel
+from pydantic import BaseModel, model_validator
 
 
 class StacSearchConfig(BaseModel):
     max_cloud_cover: float = 100.0
+    query: Optional[str] = None
     catalog_chunk_threshold: int = 10_000
     catalog_chunk_zoom: int = 5
     catalog_pagesize: int = 100
     footprint_buffer: float = 0
 
+    @model_validator(mode="before")
+    def deprecate_max_cloud_cover(cls, values: Dict[str, Any]) -> Dict[str, Any]:
+        if "max_cloud_cover" in values:  # pragma: no cover
+            raise DeprecationWarning(
+                "'max_cloud_cover' will be deprecated soon. Please use 'eo:cloud_cover<=...' in the source 'query' field.",
+            )
+        return values
+
 
 class StacStaticConfig(BaseModel):
-    max_cloud_cover: float = 100.0
+    @model_validator(mode="before")
+    def deprecate_max_cloud_cover(cls, values: Dict[str, Any]) -> Dict[str, Any]:
+        if "max_cloud_cover" in values:  # pragma: no cover
+            raise DeprecationWarning(
+                "'max_cloud_cover' will be deprecated soon. Please use 'eo:cloud_cover<=...' in the source 'query' field.",
+            )
+        return values
 
 
 class UTMSearchConfig(BaseModel):
-    max_cloud_cover: float = 100.0
+    @model_validator(mode="before")
+    def deprecate_max_cloud_cover(cls, values: Dict[str, Any]) -> Dict[str, Any]:
+        if "max_cloud_cover" in values:  # pragma: no cover
+            raise DeprecationWarning(
+                "'max_cloud_cover' will be deprecated soon. Please use 'eo:cloud_cover<=...' in the source 'query' field.",
+            )
+        return values
 
     sinergise_aws_collections: dict = dict(
         S2_L2A=dict(

mapchete_eo/search/s2_mgrs.py

@@ -6,18 +6,17 @@ from functools import cached_property
 from itertools import product
 from typing import List, Literal, Optional, Tuple, Union
 
-from mapchete.geometry import reproject_geometry
+from mapchete.geometry import (
+    reproject_geometry,
+    repair_antimeridian_geometry,
+    transform_to_latlon,
+)
 from mapchete.types import Bounds
 from rasterio.crs import CRS
 from shapely import prepare
 from shapely.geometry import box, mapping, shape
 from shapely.geometry.base import BaseGeometry
 
-from mapchete_eo.geometry import (
-    bounds_to_geom,
-    repair_antimeridian_geometry,
-    transform_to_latlon,
-)
 
 LATLON_LEFT = -180
 LATLON_RIGHT = 180
@@ -255,7 +254,7 @@ class S2Tile:
         grid_square = tile_id[3:]
         try:
             int(utm_zone)
-        except Exception:
+        except Exception:  # pragma: no cover
             raise ValueError(f"invalid UTM zone given: {utm_zone}")
 
         return MGRSCell(utm_zone, latitude_band).tile(grid_square)
@@ -268,7 +267,7 @@ class S2Tile:
 def s2_tiles_from_bounds(
     left: float, bottom: float, right: float, top: float
 ) -> List[S2Tile]:
-    bounds = Bounds(left, bottom, right, top)
+    bounds = Bounds(left, bottom, right, top, crs="EPSG:4326")
 
     # determine zones in eastern-western direction
     min_zone_idx = math.floor((left + LATLON_WIDTH_OFFSET) / UTM_ZONE_WIDTH)
@@ -291,7 +290,7 @@ def s2_tiles_from_bounds(
     min_latitude_band_idx -= 1
     max_latitude_band_idx += 1
 
-    aoi = bounds_to_geom(bounds)
+    aoi = bounds.latlon_geometry()
     prepare(aoi)
 
     def tiles_generator():

mapchete_eo/search/stac_search.py

@@ -1,19 +1,20 @@
+from __future__ import annotations
+
 import logging
 from datetime import datetime
 from functools import cached_property
-from typing import Any, Callable, Dict, Generator, Iterator, List, Optional, Set, Union
+from typing import Any, Dict, Generator, Iterator, List, Optional, Set, Union
 
 from mapchete import Timer
-from mapchete.path import MPathLike
 from mapchete.tile import BufferedTilePyramid
 from mapchete.types import Bounds, BoundsLike
 from pystac import Item
-from pystac_client import Client
-from shapely.geometry import shape
+from pystac_client import Client, CollectionClient, ItemSearch
+from shapely.geometry import shape, box
 from shapely.geometry.base import BaseGeometry
 
 from mapchete_eo.product import blacklist_products
-from mapchete_eo.search.base import CatalogSearcher, StaticCatalogWriterMixin
+from mapchete_eo.search.base import CollectionSearcher, StaticCollectionWriterMixin
 from mapchete_eo.search.config import StacSearchConfig
 from mapchete_eo.settings import mapchete_eo_settings
 from mapchete_eo.types import TimeRange
@@ -21,8 +22,8 @@ from mapchete_eo.types import TimeRange
 logger = logging.getLogger(__name__)
 
 
-class STACSearchCatalog(StaticCatalogWriterMixin, CatalogSearcher):
-    endpoint: str
+class STACSearchCollection(StaticCollectionWriterMixin, CollectionSearcher):
+    collection: str
     blacklist: Set[str] = (
         blacklist_products(mapchete_eo_settings.blacklist)
         if mapchete_eo_settings.blacklist
@@ -30,27 +31,19 @@ class STACSearchCatalog(StaticCatalogWriterMixin, CatalogSearcher):
     )
     config_cls = StacSearchConfig
 
-    def __init__(
-        self,
-        collections: Optional[List[str]] = None,
-        stac_item_modifiers: Optional[List[Callable[[Item], Item]]] = None,
-        endpoint: Optional[MPathLike] = None,
-    ):
-        if endpoint is not None:
-            self.endpoint = endpoint
-        if collections:
-            self.collections = collections
-        else:  # pragma: no cover
-            raise ValueError("collections must be given")
-        self.stac_item_modifiers = stac_item_modifiers
-
     @cached_property
-    def client(self) -> Client:
-        return Client.open(self.endpoint)
+    def client(self) -> CollectionClient:
+        return CollectionClient.from_file(self.collection)
 
     @cached_property
     def eo_bands(self) -> List[str]:
-        return self._eo_bands()
+        item_assets = self.client.extra_fields.get("item_assets", {})
+        for v in item_assets.values():
+            if "eo:bands" in v and "data" in v.get("roles", []):
+                return ["eo:bands"]
+        else:  # pragma: no cover
+            logger.debug("cannot find eo:bands definition from collections")
+            return []
 
     @cached_property
     def id(self) -> str:
@@ -69,27 +62,35 @@ class STACSearchCatalog(StaticCatalogWriterMixin, CatalogSearcher):
         time: Optional[Union[TimeRange, List[TimeRange]]] = None,
         bounds: Optional[BoundsLike] = None,
         area: Optional[BaseGeometry] = None,
+        query: Optional[str] = None,
         search_kwargs: Optional[Dict[str, Any]] = None,
     ) -> Generator[Item, None, None]:
         config = self.config_cls(**search_kwargs or {})
         if bounds:
             bounds = Bounds.from_inp(bounds)
-        if time is None:  # pragma: no cover
-            raise ValueError("time must be set")
         if area is None and bounds is None:  # pragma: no cover
             raise ValueError("either bounds or area have to be given")
 
         if area is not None and area.is_empty:  # pragma: no cover
             return
 
-        def _searches():
-            for time_range in time if isinstance(time, list) else [time]:
+        def _searches() -> Generator[ItemSearch, None, None]:
+            def _search_chunks(
+                time_range: Optional[TimeRange] = None,
+                bounds: Optional[BoundsLike] = None,
+                area: Optional[BaseGeometry] = None,
+                query: Optional[str] = None,
+            ):
                 search = self._search(
-                    time_range=time_range, bounds=bounds, area=area, config=config
+                    time_range=time_range,
+                    bounds=bounds,
+                    area=box(*area.bounds) if area else None,
+                    query=query,
+                    config=config,
                 )
                 logger.debug("found %s products", search.matched())
                 matched = search.matched() or 0
-                if matched > config.catalog_chunk_threshold:
+                if matched > config.catalog_chunk_threshold:  # pragma: no cover
                     spatial_search_chunks = SpatialSearchChunks(
                         bounds=bounds,
                         area=area,
@@ -105,6 +106,7 @@ class STACSearchCatalog(StaticCatalogWriterMixin, CatalogSearcher):
                         with Timer() as duration:
                             chunk_search = self._search(
                                 time_range=time_range,
+                                query=query,
                                 config=config,
                                 **chunk_kwargs,
                             )
@@ -119,48 +121,60 @@ class STACSearchCatalog(StaticCatalogWriterMixin, CatalogSearcher):
                 else:
                     yield search
 
-        for search in _searches():
-            for count, item in enumerate(search.items(), 1):
-                item_path = item.get_self_href()
-                # logger.debug("item %s/%s ...", count, search.matched())
-                if item_path in self.blacklist:  # pragma: no cover
-                    logger.debug("item %s found in blacklist and skipping", item_path)
-                else:
-                    yield item
+            if time:
+                # search time range(s)
+                for time_range in time if isinstance(time, list) else [time]:
+                    yield from _search_chunks(
+                        time_range=time_range,
+                        bounds=bounds,
+                        area=area,
+                        query=query,
+                    )
+            else:
+                # don't apply temporal filter
+                yield from _search_chunks(
+                    bounds=bounds,
+                    area=area,
+                    query=query,
+                )
 
-    def _eo_bands(self) -> List[str]:
-        for collection_name in self.collections:
-            collection = self.client.get_collection(collection_name)
-            if collection:
-                item_assets = collection.extra_fields.get("item_assets", {})
-                for v in item_assets.values():
-                    if "eo:bands" in v and "data" in v.get("roles", []):
-                        return ["eo:bands"]
-            else:  # pragma: no cover
-                raise ValueError(f"cannot find collection {collection}")
-        else:  # pragma: no cover
-            logger.debug("cannot find eo:bands definition from collections")
-            return []
+        for search in _searches():
+            for item in search.items():
+                if item.get_self_href() in self.blacklist:  # pragma: no cover
+                    logger.debug(
+                        "item %s found in blacklist and skipping", item.get_self_href()
+                    )
+                    continue
+                yield item
 
     @cached_property
     def default_search_params(self):
         return {
-            "collections": self.collections,
+            "collections": [self.client],
             "bbox": None,
             "intersects": None,
         }
 
+    @cached_property
+    def search_client(self) -> Client:
+        # looks weird, right?
+        #
+        # one would assume that directly returning self.client.get_root() would
+        # do the same but if we do so, it seems to ignore the "collections" parameter
+        # and thus query all collection available on that search endpoint.
+        #
+        # the only way to fix this, is to instantiate Client from scratch.
+        return Client.from_file(self.client.get_root().self_href)
+
     def _search(
         self,
         time_range: Optional[TimeRange] = None,
         bounds: Optional[Bounds] = None,
         area: Optional[BaseGeometry] = None,
+        query: Optional[str] = None,
         config: StacSearchConfig = StacSearchConfig(),
         **kwargs,
-    ):
-        if time_range is None:  # pragma: no cover
-            raise ValueError("time_range not provided")
-
+    ) -> ItemSearch:
         if bounds is not None:
             if shape(bounds).is_empty:  # pragma: no cover
                 raise ValueError("bounds empty")
@@ -170,22 +184,29 @@ class STACSearchCatalog(StaticCatalogWriterMixin, CatalogSearcher):
                 raise ValueError("area empty")
             kwargs.update(intersects=area)
 
-        start = (
-            time_range.start.date()
-            if isinstance(time_range.start, datetime)
-            else time_range.start
-        )
-        end = (
-            time_range.end.date()
-            if isinstance(time_range.end, datetime)
-            else time_range.end
-        )
-        search_params = dict(
-            self.default_search_params,
-            datetime=f"{start}/{end}",
-            query=[f"eo:cloud_cover<={config.max_cloud_cover}"],
-            **kwargs,
-        )
+        if time_range:
+            start = (
+                time_range.start.date()
+                if isinstance(time_range.start, datetime)
+                else time_range.start
+            )
+            end = (
+                time_range.end.date()
+                if isinstance(time_range.end, datetime)
+                else time_range.end
+            )
+            search_params = dict(
+                self.default_search_params,
+                datetime=f"{start}/{end}",
+                query=[query] if query else None,
+                **kwargs,
+            )
+        else:
+            search_params = dict(
+                self.default_search_params,
+                query=[query] if query else None,
+                **kwargs,
+            )
         if (
             bounds is None
             and area is None
@@ -194,14 +215,12 @@ class STACSearchCatalog(StaticCatalogWriterMixin, CatalogSearcher):
             raise ValueError("no bounds or area given")
         logger.debug("query catalog using params: %s", search_params)
         with Timer() as duration:
-            result = self.client.search(**search_params, limit=config.catalog_pagesize)
+            result = self.search_client.search(
+                **search_params, limit=config.catalog_pagesize
+            )
         logger.debug("query took %s", str(duration))
         return result
 
-    def get_collections(self):
-        for collection_name in self.collections:
-            yield self.client.get_collection(collection_name)
-
 
 class SpatialSearchChunks:
     bounds: Bounds