mapchete-eo 2025.10.1__py2.py3-none-any.whl → 2026.1.0__py2.py3-none-any.whl

mapchete_eo/protocols.py

@@ -15,6 +15,7 @@ from mapchete.io.raster import ReferencedRaster
 
 
 class EOProductProtocol(Protocol):
+    id: str
     bounds: Bounds
     crs: CRS
     __geo_interface__: Optional[Dict[str, Any]]
@@ -54,3 +55,7 @@ class EOProductProtocol(Protocol):
 
 class DateTimeProtocol(Protocol):
     datetime: DateTimeLike
+
+
+class GetPropertyProtocol(Protocol):
+    def get_property(self, property: str) -> Any: ...

mapchete_eo/search/__init__.py

@@ -7,8 +7,8 @@ of product metadata.
 It helps the InputData class to find the input products and their metadata.
 """
 
-from mapchete_eo.search.stac_search import STACSearchCatalog
-from mapchete_eo.search.stac_static import STACStaticCatalog
+from mapchete_eo.search.stac_search import STACSearchCollection
+from mapchete_eo.search.stac_static import STACStaticCollection
 from mapchete_eo.search.utm_search import UTMSearchCatalog
 
-__all__ = ["STACSearchCatalog", "STACStaticCatalog", "UTMSearchCatalog"]
+__all__ = ["STACSearchCollection", "STACStaticCollection", "UTMSearchCatalog"]

mapchete_eo/search/base.py

@@ -2,20 +2,24 @@ from functools import cached_property
 import json
 import logging
 from abc import ABC, abstractmethod
-from typing import Any, Callable, Dict, Generator, List, Optional, Type, Union
+from typing import Any, Callable, Dict, Generator, List, Optional, Set, Type, Union
 
+from pygeofilter.parsers.ecql import parse as parse_ecql
+from pygeofilter.backends.native.evaluate import NativeEvaluator
 from pydantic import BaseModel
-from pystac import Item, Catalog, CatalogType, Extent
 from mapchete.path import MPath, MPathLike
 from mapchete.types import Bounds
+from pystac import Catalog, Item, CatalogType, Extent
 from pystac.collection import Collection
 from pystac.stac_io import DefaultStacIO
-from pystac_client import Client
+from pystac_client import CollectionClient
 from pystac_client.stac_api_io import StacApiIO
 from rasterio.profiles import Profile
 from shapely.geometry.base import BaseGeometry
 
 from mapchete_eo.io.assets import get_assets, get_metadata_assets
+from mapchete_eo.product import blacklist_products
+from mapchete_eo.settings import mapchete_eo_settings
 from mapchete_eo.types import TimeRange
 
 logger = logging.getLogger(__name__)
@@ -44,29 +48,51 @@ class FSSpecStacIO(StacApiIO):
             return dst.write(json.dumps(json_dict, indent=2))
 
 
-class CatalogSearcher(ABC):
+class CollectionSearcher(ABC):
     """
     This class serves as a bridge between an Archive and a catalog implementation.
     """
 
-    collections: List[str]
     config_cls: Type[BaseModel]
+    collection: str
+    stac_item_modifiers: Optional[List[Callable[[Item], Item]]] = None
+    blacklist: Set[str] = (
+        blacklist_products(mapchete_eo_settings.blacklist)
+        if mapchete_eo_settings.blacklist
+        else set()
+    )
+
+    def __init__(
+        self,
+        collection: str,
+        stac_item_modifiers: Optional[List[Callable[[Item], Item]]] = None,
+    ):
+        self.collection = collection
+        self.stac_item_modifiers = stac_item_modifiers
 
     @abstractmethod
     @cached_property
-    def eo_bands(self) -> List[str]: ...
+    def client(self) -> CollectionClient: ...
 
     @abstractmethod
     @cached_property
-    def id(self) -> str: ...
+    def eo_bands(self) -> List[str]: ...
+
+    @property
+    def config(self) -> BaseModel:
+        return self.config_cls()
 
-    @abstractmethod
     @cached_property
-    def description(self) -> str: ...
+    def id(self) -> str:
+        return self.client.id
+
+    @cached_property
+    def description(self) -> str:
+        return self.client.description
 
-    @abstractmethod
     @cached_property
-    def stac_extensions(self) -> List[str]: ...
+    def stac_extensions(self) -> List[str]:
+        return self.client.stac_extensions
 
     @abstractmethod
     def search(
@@ -74,20 +100,17 @@ class CatalogSearcher(ABC):
         time: Optional[Union[TimeRange, List[TimeRange]]] = None,
         bounds: Optional[Bounds] = None,
         area: Optional[BaseGeometry] = None,
+        query: Optional[str] = None,
         search_kwargs: Optional[Dict[str, Any]] = None,
     ) -> Generator[Item, None, None]: ...
 
 
-class StaticCatalogWriterMixin(CatalogSearcher):
+class StaticCollectionWriterMixin(CollectionSearcher):
     # client: Client
     # id: str
     # description: str
     # stac_extensions: List[str]
 
-    @abstractmethod
-    def get_collections(self) -> List[Collection]:  # pragma: no cover
-        ...
-
     def write_static_catalog(
         self,
         output_path: MPathLike,
@@ -113,94 +136,93 @@ class StaticCatalogWriterMixin(CatalogSearcher):
         catalog_json = output_path / "catalog.json"
         if catalog_json.exists():
             logger.debug("open existing catalog %s", str(catalog_json))
-            client = Client.from_file(catalog_json)
-            # catalog = pystac.Catalog.from_file(catalog_json)
-            existing_collections = list(client.get_collections())
+            catalog = Catalog.from_file(catalog_json)
+            # client = Client.from_file(catalog_json)
+            # existing_collection = client.get_collection(self.id)
         else:
-            existing_collections = []
-        catalog = Catalog(
-            name or f"{self.id}",
-            description or f"Static subset of {self.description}",
-            stac_extensions=self.stac_extensions,
-            href=str(catalog_json),
-            catalog_type=CatalogType.SELF_CONTAINED,
-        )
+            # existing_collections = []
+            catalog = Catalog(
+                name or f"{self.id}",
+                description or f"Static subset of {self.description}",
+                stac_extensions=self.stac_extensions,
+                href=str(catalog_json),
+                catalog_type=CatalogType.SELF_CONTAINED,
+            )
         src_items = list(
             self.search(
                 time=time, bounds=bounds, area=area, search_kwargs=search_kwargs
             )
         )
-        for collection in self.get_collections():
-            # collect all items and download assets if required
-            items: List[Item] = []
-            item_ids = set()
-            for n, item in enumerate(src_items, 1):
-                logger.debug("found item %s", item)
-                item = item.clone()
-                if assets:
-                    logger.debug("get assets %s", assets)
-                    item = get_assets(
-                        item,
-                        assets,
-                        output_path / collection.id / item.id,
-                        resolution=assets_dst_resolution,
-                        convert_profile=assets_convert_profile,
-                        overwrite=overwrite,
-                        ignore_if_exists=True,
-                    )
-                if copy_metadata:
-                    item = get_metadata_assets(
-                        item,
-                        output_path / collection.id / item.id,
-                        metadata_parser_classes=metadata_parser_classes,
-                        resolution=assets_dst_resolution,
-                        convert_profile=assets_convert_profile,
-                        overwrite=overwrite,
-                    )
-                # this has to be set to None, otherwise pystac will mess up the asset paths
-                # after normalizing
-                item.set_self_href(None)
-
-                items.append(item)
-                item_ids.add(item.id)
-
-                if progress_callback:
-                    progress_callback(n=n, total=len(src_items))
-
-            for existing_collection in existing_collections:
-                if existing_collection.id == collection.id:
-                    logger.debug("try to find unregistered items in collection")
-                    collection_root_path = MPath.from_inp(
-                        existing_collection.get_self_href()
-                    ).parent
-                    for subpath in collection_root_path.ls():
-                        if subpath.is_directory():
-                            try:
-                                item = Item.from_file(
-                                    subpath / subpath.with_suffix(".json").name
-                                )
-                                if item.id not in item_ids:
-                                    logger.debug(
-                                        "add existing item with id %s", item.id
-                                    )
-                                    items.append(item)
-                                    item_ids.add(item.id)
-                            except FileNotFoundError:
-                                pass
-                    break
+        # collect all items and download assets if required
+        items: List[Item] = []
+        item_ids = set()
+        for n, item in enumerate(src_items, 1):
+            logger.debug("found item %s", item)
+            item = item.clone()
+            if assets:
+                logger.debug("get assets %s", assets)
+                item = get_assets(
+                    item,
+                    assets,
+                    output_path / self.id / item.id,
+                    resolution=assets_dst_resolution,
+                    convert_profile=assets_convert_profile,
+                    overwrite=overwrite,
+                    ignore_if_exists=True,
+                )
+            if copy_metadata:
+                item = get_metadata_assets(
+                    item,
+                    output_path / self.id / item.id,
+                    metadata_parser_classes=metadata_parser_classes,
+                    resolution=assets_dst_resolution,
+                    convert_profile=assets_convert_profile,
+                    overwrite=overwrite,
+                )
+            # this has to be set to None, otherwise pystac will mess up the asset paths
+            # after normalizing
+            item.set_self_href(None)
+
+            items.append(item)
+            item_ids.add(item.id)
+
+            if progress_callback:
+                progress_callback(n=n, total=len(src_items))
+
+            # for existing_collection in existing_collections:
+            #     if existing_collection.id == collection.id:
+            #         logger.debug("try to find unregistered items in collection")
+            #         collection_root_path = MPath.from_inp(
+            #             existing_collection.get_self_href()
+            #         ).parent
+            #         for subpath in collection_root_path.ls():
+            #             if subpath.is_directory():
+            #                 try:
+            #                     item = Item.from_file(
+            #                         subpath / subpath.with_suffix(".json").name
+            #                     )
+            #                     if item.id not in item_ids:
+            #                         logger.debug(
+            #                             "add existing item with id %s", item.id
+            #                         )
+            #                         items.append(item)
+            #                         item_ids.add(item.id)
+            #                 except FileNotFoundError:
+            #                     pass
+            #         break
             # create collection and copy metadata
             logger.debug("create new collection")
             out_collection = Collection(
-                id=collection.id,
+                id=self.id,
                 extent=Extent.from_items(items),
-                description=collection.description,
-                title=collection.title,
-                stac_extensions=collection.stac_extensions,
-                license=collection.license,
-                keywords=collection.keywords,
-                providers=collection.providers,
-                summaries=collection.summaries,
-                extra_fields=collection.extra_fields,
+                description=self.description,
+                title=self.client.title,
+                stac_extensions=self.stac_extensions,
+                license=self.client.license,
+                keywords=self.client.keywords,
+                providers=self.client.providers,
+                summaries=self.client.summaries,
+                extra_fields=self.client.extra_fields,
                 catalog_type=CatalogType.SELF_CONTAINED,
             )
 
@@ -222,14 +244,20 @@ class StaticCatalogWriterMixin(CatalogSearcher):
 
 def filter_items(
     items: Generator[Item, None, None],
-    cloud_cover_field: str = "eo:cloud_cover",
-    max_cloud_cover: float = 100.0,
+    query: Optional[str] = None,
 ) -> Generator[Item, None, None]:
     """
     Only for cloudcover now, this can and should be adapted for filter field and value
     the field and value for the item filter would be defined in search.config.py corresponding configs
     and passed down to the individual search approaches via said config and this Function.
     """
-    for item in items:
-        if item.properties.get(cloud_cover_field, 0.0) <= max_cloud_cover:
-            yield item
+    if query:
+        ast = parse_ecql(query)
+        evaluator = NativeEvaluator(use_getattr=False)
+        filter_func = evaluator.evaluate(ast)
+        for item in items:
+            # pystac items store metadata in 'properties'
+            if filter_func(item.properties):
+                yield item
+    else:
+        yield from items

mapchete_eo/search/config.py

@@ -1,23 +1,48 @@
-from typing import Optional
+import logging
+
+
+from contextlib import contextmanager
+from typing import Optional, Dict, Any
 
 from mapchete.path import MPath, MPathLike
-from pydantic import BaseModel
+from pydantic import BaseModel, model_validator
 
 
 class StacSearchConfig(BaseModel):
     max_cloud_cover: float = 100.0
+    query: Optional[str] = None
     catalog_chunk_threshold: int = 10_000
     catalog_chunk_zoom: int = 5
     catalog_pagesize: int = 100
     footprint_buffer: float = 0
 
+    @model_validator(mode="before")
+    def deprecate_max_cloud_cover(cls, values: Dict[str, Any]) -> Dict[str, Any]:
+        if "max_cloud_cover" in values:  # pragma: no cover
+            raise DeprecationWarning(
+                "'max_cloud_cover' will be deprecated soon. Please use 'eo:cloud_cover<=...' in the source 'query' field.",
+            )
+        return values
+
 
 class StacStaticConfig(BaseModel):
-    max_cloud_cover: float = 100.0
+    @model_validator(mode="before")
+    def deprecate_max_cloud_cover(cls, values: Dict[str, Any]) -> Dict[str, Any]:
+        if "max_cloud_cover" in values:  # pragma: no cover
+            raise DeprecationWarning(
+                "'max_cloud_cover' will be deprecated soon. Please use 'eo:cloud_cover<=...' in the source 'query' field.",
+            )
+        return values
 
 
 class UTMSearchConfig(BaseModel):
-    max_cloud_cover: float = 100.0
+    @model_validator(mode="before")
+    def deprecate_max_cloud_cover(cls, values: Dict[str, Any]) -> Dict[str, Any]:
+        if "max_cloud_cover" in values:  # pragma: no cover
+            raise DeprecationWarning(
+                "'max_cloud_cover' will be deprecated soon. Please use 'eo:cloud_cover<=...' in the source 'query' field.",
+            )
+        return values
 
     sinergise_aws_collections: dict = dict(
         S2_L2A=dict(
@@ -25,18 +50,64 @@ class UTMSearchConfig(BaseModel):
             path=MPath(
                 "https://sentinel-s2-l2a-stac.s3.amazonaws.com/sentinel-s2-l2a.json"
             ),
+            endpoint="s3://sentinel-s2-l2a-stac",
         ),
         S2_L1C=dict(
             id="sentinel-s2-l1c",
             path=MPath(
                 "https://sentinel-s2-l1c-stac.s3.amazonaws.com/sentinel-s2-l1c.json"
             ),
+            endpoint="s3://sentinel-s2-l1c-stac",
         ),
         S1_GRD=dict(
             id="sentinel-s1-l1c",
             path=MPath(
                 "https://sentinel-s1-l1c-stac.s3.amazonaws.com/sentinel-s1-l1c.json"
             ),
+            endpoint="s3://sentinel-s1-l1c-stac",
         ),
     )
     search_index: Optional[MPathLike] = None
+
+
+@contextmanager
+def patch_invalid_assets():
+    """
+    Context manager/decorator to fix pystac crash on malformed assets (strings instead of dicts).
+
+    """
+    try:
+        from pystac.extensions.file import FileExtensionHooks
+    except ImportError:  # pragma: no cover
+        yield
+        return
+
+    logger = logging.getLogger(__name__)
+
+    _original_migrate = FileExtensionHooks.migrate
+
+    def _safe_migrate(self, obj, version, info):
+        if "assets" in obj and isinstance(obj["assets"], dict):
+            bad_keys = []
+            for key, asset in obj["assets"].items():
+                if not isinstance(asset, dict):
+                    logger.debug(
+                        "Removing malformed asset '%s' (type %s) from item %s",
+                        key,
+                        type(asset),
+                        obj.get("id", "unknown"),
+                    )
+                    bad_keys.append(key)
+
+            for key in bad_keys:
+                del obj["assets"][key]
+
+        return _original_migrate(self, obj, version, info)
+
+    # Apply patch
+    FileExtensionHooks.migrate = _safe_migrate
+    try:
+        yield
+    finally:
+        # Restore original
+        FileExtensionHooks.migrate = _original_migrate

mapchete_eo/search/s2_mgrs.py

@@ -6,18 +6,17 @@ from functools import cached_property
 from itertools import product
 from typing import List, Literal, Optional, Tuple, Union
 
-from mapchete.geometry import reproject_geometry
+from mapchete.geometry import (
+    reproject_geometry,
+    repair_antimeridian_geometry,
+    transform_to_latlon,
+)
 from mapchete.types import Bounds
 from rasterio.crs import CRS
 from shapely import prepare
 from shapely.geometry import box, mapping, shape
 from shapely.geometry.base import BaseGeometry
 
-from mapchete_eo.geometry import (
-    bounds_to_geom,
-    repair_antimeridian_geometry,
-    transform_to_latlon,
-)
 
 LATLON_LEFT = -180
 LATLON_RIGHT = 180
@@ -255,7 +254,7 @@ class S2Tile:
         grid_square = tile_id[3:]
         try:
             int(utm_zone)
-        except Exception:
+        except Exception:  # pragma: no cover
             raise ValueError(f"invalid UTM zone given: {utm_zone}")
 
         return MGRSCell(utm_zone, latitude_band).tile(grid_square)
@@ -268,7 +267,7 @@ class S2Tile:
 def s2_tiles_from_bounds(
     left: float, bottom: float, right: float, top: float
 ) -> List[S2Tile]:
-    bounds = Bounds(left, bottom, right, top)
+    bounds = Bounds(left, bottom, right, top, crs="EPSG:4326")
 
     # determine zones in eastern-western direction
     min_zone_idx = math.floor((left + LATLON_WIDTH_OFFSET) / UTM_ZONE_WIDTH)
@@ -291,7 +290,7 @@ def s2_tiles_from_bounds(
     min_latitude_band_idx -= 1
     max_latitude_band_idx += 1
 
-    aoi = bounds_to_geom(bounds)
+    aoi = bounds.latlon_geometry()
     prepare(aoi)
 
     def tiles_generator():