mapchete-eo 2025.10.0__py2.py3-none-any.whl → 2025.11.0__py2.py3-none-any.whl

mapchete_eo/search/stac_search.py

@@ -1,19 +1,20 @@
+from __future__ import annotations
+
 import logging
 from datetime import datetime
 from functools import cached_property
-from typing import Any, Callable, Dict, Generator, Iterator, List, Optional, Set, Union
+from typing import Any, Dict, Generator, Iterator, List, Optional, Set, Union
 
 from mapchete import Timer
-from mapchete.path import MPathLike
 from mapchete.tile import BufferedTilePyramid
 from mapchete.types import Bounds, BoundsLike
 from pystac import Item
-from pystac_client import Client
-from shapely.geometry import shape
+from pystac_client import Client, CollectionClient, ItemSearch
+from shapely.geometry import shape, box
 from shapely.geometry.base import BaseGeometry
 
 from mapchete_eo.product import blacklist_products
-from mapchete_eo.search.base import CatalogSearcher, StaticCatalogWriterMixin
+from mapchete_eo.search.base import CollectionSearcher, StaticCollectionWriterMixin
 from mapchete_eo.search.config import StacSearchConfig
 from mapchete_eo.settings import mapchete_eo_settings
 from mapchete_eo.types import TimeRange
@@ -21,8 +22,8 @@ from mapchete_eo.types import TimeRange
 logger = logging.getLogger(__name__)
 
 
-class STACSearchCatalog(StaticCatalogWriterMixin, CatalogSearcher):
-    endpoint: str
+class STACSearchCollection(StaticCollectionWriterMixin, CollectionSearcher):
+    collection: str
     blacklist: Set[str] = (
         blacklist_products(mapchete_eo_settings.blacklist)
         if mapchete_eo_settings.blacklist
@@ -30,49 +31,66 @@ class STACSearchCatalog(StaticCatalogWriterMixin, CatalogSearcher):
     )
     config_cls = StacSearchConfig
 
-    def __init__(
-        self,
-        collections: Optional[List[str]] = None,
-        stac_item_modifiers: Optional[List[Callable[[Item], Item]]] = None,
-        endpoint: Optional[MPathLike] = None,
-    ):
-        if collections:
-            self.collections = collections
+    @cached_property
+    def client(self) -> CollectionClient:
+        return CollectionClient.from_file(self.collection)
+
+    @cached_property
+    def eo_bands(self) -> List[str]:
+        item_assets = self.client.extra_fields.get("item_assets", {})
+        for v in item_assets.values():
+            if "eo:bands" in v and "data" in v.get("roles", []):
+                return ["eo:bands"]
         else:  # pragma: no cover
-            raise ValueError("collections must be given")
-        self.client = Client.open(endpoint or self.endpoint)
-        self.id = self.client.id
-        self.description = self.client.description
-        self.stac_extensions = self.client.stac_extensions
-        self.eo_bands = self._eo_bands()
-        self.stac_item_modifiers = stac_item_modifiers
+            logger.debug("cannot find eo:bands definition from collections")
+            return []
+
+    @cached_property
+    def id(self) -> str:
+        return self.client.id
+
+    @cached_property
+    def description(self) -> str:
+        return self.client.description
+
+    @cached_property
+    def stac_extensions(self) -> List[str]:
+        return self.client.stac_extensions
 
     def search(
         self,
         time: Optional[Union[TimeRange, List[TimeRange]]] = None,
         bounds: Optional[BoundsLike] = None,
         area: Optional[BaseGeometry] = None,
+        query: Optional[str] = None,
         search_kwargs: Optional[Dict[str, Any]] = None,
     ) -> Generator[Item, None, None]:
         config = self.config_cls(**search_kwargs or {})
         if bounds:
             bounds = Bounds.from_inp(bounds)
-        if time is None:  # pragma: no cover
-            raise ValueError("time must be set")
         if area is None and bounds is None:  # pragma: no cover
             raise ValueError("either bounds or area have to be given")
 
         if area is not None and area.is_empty:  # pragma: no cover
             return
 
-        def _searches():
-            for time_range in time if isinstance(time, list) else [time]:
+        def _searches() -> Generator[ItemSearch, None, None]:
+            def _search_chunks(
+                time_range: Optional[TimeRange] = None,
+                bounds: Optional[BoundsLike] = None,
+                area: Optional[BaseGeometry] = None,
+                query: Optional[str] = None,
+            ):
                 search = self._search(
-                    time_range=time_range, bounds=bounds, area=area, config=config
+                    time_range=time_range,
+                    bounds=bounds,
+                    area=box(*area.bounds) if area else None,
+                    query=query,
+                    config=config,
                 )
                 logger.debug("found %s products", search.matched())
                 matched = search.matched() or 0
-                if matched > config.catalog_chunk_threshold:
+                if matched > config.catalog_chunk_threshold:  # pragma: no cover
                     spatial_search_chunks = SpatialSearchChunks(
                         bounds=bounds,
                         area=area,
@@ -88,6 +106,7 @@ class STACSearchCatalog(StaticCatalogWriterMixin, CatalogSearcher):
                         with Timer() as duration:
                             chunk_search = self._search(
                                 time_range=time_range,
+                                query=query,
                                 config=config,
                                 **chunk_kwargs,
                             )
@@ -102,48 +121,60 @@ class STACSearchCatalog(StaticCatalogWriterMixin, CatalogSearcher):
                 else:
                     yield search
 
+            if time:
+                # search time range(s)
+                for time_range in time if isinstance(time, list) else [time]:
+                    yield from _search_chunks(
+                        time_range=time_range,
+                        bounds=bounds,
+                        area=area,
+                        query=query,
+                    )
+            else:
+                # don't apply temporal filter
+                yield from _search_chunks(
+                    bounds=bounds,
+                    area=area,
+                    query=query,
+                )
+
         for search in _searches():
-            for count, item in enumerate(search.items(), 1):
-                item_path = item.get_self_href()
-                # logger.debug("item %s/%s ...", count, search.matched())
-                if item_path in self.blacklist:  # pragma: no cover
-                    logger.debug("item %s found in blacklist and skipping", item_path)
-                else:
-                    yield item
-
-    def _eo_bands(self) -> List[str]:
-        for collection_name in self.collections:
-            collection = self.client.get_collection(collection_name)
-            if collection:
-                item_assets = collection.extra_fields.get("item_assets", {})
-                for v in item_assets.values():
-                    if "eo:bands" in v and "data" in v.get("roles", []):
-                        return ["eo:bands"]
-            else:  # pragma: no cover
-                raise ValueError(f"cannot find collection {collection}")
-        else:  # pragma: no cover
-            logger.debug("cannot find eo:bands definition from collections")
-            return []
+            for item in search.items():
+                if item.get_self_href() in self.blacklist:  # pragma: no cover
+                    logger.debug(
+                        "item %s found in blacklist and skipping", item.get_self_href()
+                    )
+                    continue
+                yield item
 
     @cached_property
     def default_search_params(self):
         return {
-            "collections": self.collections,
+            "collections": [self.client],
             "bbox": None,
             "intersects": None,
         }
 
+    @cached_property
+    def search_client(self) -> Client:
+        # looks weird, right?
+        #
+        # one would assume that directly returning self.client.get_root() would
+        # do the same but if we do so, it seems to ignore the "collections" parameter
+        # and thus query all collection available on that search endpoint.
+        #
+        # the only way to fix this, is to instantiate Client from scratch.
+        return Client.from_file(self.client.get_root().self_href)
+
     def _search(
         self,
         time_range: Optional[TimeRange] = None,
         bounds: Optional[Bounds] = None,
         area: Optional[BaseGeometry] = None,
+        query: Optional[str] = None,
         config: StacSearchConfig = StacSearchConfig(),
         **kwargs,
-    ):
-        if time_range is None:  # pragma: no cover
-            raise ValueError("time_range not provided")
-
+    ) -> ItemSearch:
         if bounds is not None:
             if shape(bounds).is_empty:  # pragma: no cover
                 raise ValueError("bounds empty")
@@ -153,22 +184,29 @@ class STACSearchCatalog(StaticCatalogWriterMixin, CatalogSearcher):
                 raise ValueError("area empty")
             kwargs.update(intersects=area)
 
-        start = (
-            time_range.start.date()
-            if isinstance(time_range.start, datetime)
-            else time_range.start
-        )
-        end = (
-            time_range.end.date()
-            if isinstance(time_range.end, datetime)
-            else time_range.end
-        )
-        search_params = dict(
-            self.default_search_params,
-            datetime=f"{start}/{end}",
-            query=[f"eo:cloud_cover<={config.max_cloud_cover}"],
-            **kwargs,
-        )
+        if time_range:
+            start = (
+                time_range.start.date()
+                if isinstance(time_range.start, datetime)
+                else time_range.start
+            )
+            end = (
+                time_range.end.date()
+                if isinstance(time_range.end, datetime)
+                else time_range.end
+            )
+            search_params = dict(
+                self.default_search_params,
+                datetime=f"{start}/{end}",
+                query=[query] if query else None,
+                **kwargs,
+            )
+        else:
+            search_params = dict(
+                self.default_search_params,
+                query=[query] if query else None,
+                **kwargs,
+            )
         if (
             bounds is None
             and area is None
@@ -177,14 +215,12 @@ class STACSearchCatalog(StaticCatalogWriterMixin, CatalogSearcher):
             raise ValueError("no bounds or area given")
         logger.debug("query catalog using params: %s", search_params)
         with Timer() as duration:
-            result = self.client.search(**search_params, limit=config.catalog_pagesize)
+            result = self.search_client.search(
+                **search_params, limit=config.catalog_pagesize
+            )
         logger.debug("query took %s", str(duration))
         return result
 
-    def get_collections(self):
-        for collection_name in self.collections:
-            yield self.client.get_collection(collection_name)
-
 
 class SpatialSearchChunks:
     bounds: Bounds

mapchete_eo/search/stac_static.py

@@ -1,21 +1,21 @@
+from functools import cached_property
 import logging
 import warnings
-from typing import Any, Callable, Dict, Generator, List, Optional, Union
+from typing import Any, Dict, Generator, List, Optional, Union
 
 from mapchete import Bounds
 from mapchete.types import BoundsLike
 from pystac import Item, Catalog, Collection
 from mapchete.io.vector import bounds_intersect
-from mapchete.path import MPathLike
 from pystac.stac_io import StacIO
-from pystac_client import Client
+from pystac_client import CollectionClient
 from shapely.geometry import shape
 from shapely.geometry.base import BaseGeometry
 
 from mapchete_eo.search.base import (
-    CatalogSearcher,
+    CollectionSearcher,
     FSSpecStacIO,
-    StaticCatalogWriterMixin,
+    StaticCollectionWriterMixin,
     filter_items,
 )
 from mapchete_eo.search.config import StacStaticConfig
@@ -28,37 +28,67 @@ logger = logging.getLogger(__name__)
 StacIO.set_default(FSSpecStacIO)
 
 
-class STACStaticCatalog(StaticCatalogWriterMixin, CatalogSearcher):
+class STACStaticCollection(StaticCollectionWriterMixin, CollectionSearcher):
     config_cls = StacStaticConfig
 
-    def __init__(
-        self,
-        baseurl: MPathLike,
-        stac_item_modifiers: Optional[List[Callable[[Item], Item]]] = None,
-    ):
-        self.client = Client.from_file(str(baseurl), stac_io=FSSpecStacIO())
-        self.id = self.client.id
-        self.description = self.client.description
-        self.stac_extensions = self.client.stac_extensions
-        self.collections = [c.id for c in self.client.get_children()]
-        self.eo_bands = self._eo_bands()
-        self.stac_item_modifiers = stac_item_modifiers
+    @cached_property
+    def client(self) -> CollectionClient:
+        return CollectionClient.from_file(str(self.collection), stac_io=FSSpecStacIO())
+
+    @cached_property
+    def eo_bands(self) -> List[str]:
+        eo_bands = self.client.extra_fields.get("properties", {}).get("eo:bands")
+        if eo_bands:
+            return eo_bands
+        else:
+            warnings.warn(
+                "Unable to read eo:bands definition from collection. "
+                "Trying now to get information from assets ..."
+            )
+            # see if eo:bands can be found in properties
+            try:
+                item = next(self.client.get_items(recursive=True))
+                eo_bands = item.properties.get("eo:bands")
+                if eo_bands:
+                    return eo_bands
+
+                # look through the assets and collect eo:bands
+                out = {}
+                for asset in item.assets.values():
+                    for eo_band in asset.extra_fields.get("eo:bands", []):
+                        out[eo_band["name"]] = eo_band
+                if out:
+                    return [v for v in out.values()]
+            except StopIteration:
+                pass
+
+            logger.debug("cannot find eo:bands definition")
+            return []
+
+    @cached_property
+    def id(self) -> str:
+        return self.client.id
+
+    @cached_property
+    def description(self) -> str:
+        return self.client.description
+
+    @cached_property
+    def stac_extensions(self) -> List[str]:
+        return self.client.stac_extensions
 
     def search(
         self,
         time: Optional[Union[TimeRange, List[TimeRange]]] = None,
         bounds: Optional[BoundsLike] = None,
         area: Optional[BaseGeometry] = None,
+        query: Optional[str] = None,
         search_kwargs: Optional[Dict[str, Any]] = None,
     ) -> Generator[Item, None, None]:
-        config = self.config_cls(**search_kwargs or {})
         if area is None and bounds:
             bounds = Bounds.from_inp(bounds)
             area = shape(bounds)
-        for item in filter_items(
-            self._raw_search(time=time, area=area),
-            max_cloud_cover=config.max_cloud_cover,
-        ):
+        for item in filter_items(self._raw_search(time=time, area=area), query=query):
             yield item
 
     def _raw_search(
@@ -69,83 +99,22 @@ class STACStaticCatalog(StaticCatalogWriterMixin, CatalogSearcher):
         if area is not None and area.is_empty:
             return
         logger.debug("iterate through children")
-        for collection in self.client.get_collections():
-            if time:
-                for time_range in time if isinstance(time, list) else [time]:
-                    for item in _all_intersecting_items(
-                        collection,
-                        area=area,
-                        time_range=time_range,
-                    ):
-                        item.make_asset_hrefs_absolute()
-                        yield item
-            else:
+        if time:
+            for time_range in time if isinstance(time, list) else [time]:
                 for item in _all_intersecting_items(
-                    collection,
+                    self.client,
                     area=area,
+                    time_range=time_range,
                 ):
                     item.make_asset_hrefs_absolute()
                     yield item
-
-    def _eo_bands(self) -> List[str]:
-        for collection in self.client.get_children():
-            eo_bands = collection.extra_fields.get("properties", {}).get("eo:bands")
-            if eo_bands:
-                return eo_bands
         else:
-            warnings.warn(
-                "Unable to read eo:bands definition from collections. "
-                "Trying now to get information from assets ..."
-            )
-
-            # see if eo:bands can be found in properties
-            item = _get_first_item(self.client.get_children())
-            eo_bands = item.properties.get("eo:bands")
-            if eo_bands:
-                return eo_bands
-
-            # look through the assets and collect eo:bands
-            out = {}
-            for asset in item.assets.values():
-                for eo_band in asset.extra_fields.get("eo:bands", []):
-                    out[eo_band["name"]] = eo_band
-            if out:
-                return [v for v in out.values()]
-
-            logger.debug("cannot find eo:bands definition")
-            return []
-
-    def get_collections(
-        self,
-        time: Optional[Union[TimeRange, List[TimeRange]]] = None,
-        bounds: Optional[BoundsLike] = None,
-        area: Optional[BaseGeometry] = None,
-    ):
-        if area is None and bounds is not None:
-            area = Bounds.from_inp(bounds).geometry
-        for collection in self.client.get_children():
-            if time:
-                for time_range in time if isinstance(time, list) else [time]:
-                    if _collection_extent_intersects(
-                        collection,
-                        area=area,
-                        time_range=time_range,
-                    ):
-                        yield collection
-            else:
-                if _collection_extent_intersects(collection, area=area):
-                    yield collection
-
-
-def _get_first_item(collections):
-    for collection in collections:
-        for item in collection.get_all_items():
-            return item
-        else:
-            for child in collection.get_children():
-                return _get_first_item(child)
-    else:
-        raise ValueError("collections contain no items")
+            for item in _all_intersecting_items(
+                self.client,
+                area=area,
+            ):
+                item.make_asset_hrefs_absolute()
+                yield item
 
 
 def _all_intersecting_items(

mapchete_eo/search/utm_search.py

@@ -1,6 +1,7 @@
 import datetime
+from functools import cached_property
 import logging
-from typing import Any, Callable, Dict, Generator, List, Optional, Set, Union
+from typing import Any, Dict, Generator, List, Optional, Set, Union
 
 from mapchete.io.vector import fiona_open
 from mapchete.path import MPath, MPathLike
@@ -14,8 +15,8 @@ from shapely.geometry.base import BaseGeometry
 from mapchete_eo.exceptions import ItemGeometryError
 from mapchete_eo.product import blacklist_products
 from mapchete_eo.search.base import (
-    CatalogSearcher,
-    StaticCatalogWriterMixin,
+    CollectionSearcher,
+    StaticCollectionWriterMixin,
     filter_items,
 )
 from mapchete_eo.search.config import UTMSearchConfig
@@ -27,7 +28,7 @@ from mapchete_eo.types import TimeRange
 logger = logging.getLogger(__name__)
 
 
-class UTMSearchCatalog(StaticCatalogWriterMixin, CatalogSearcher):
+class UTMSearchCatalog(StaticCollectionWriterMixin, CollectionSearcher):
     endpoint: str
     id: str
     day_subdir_schema: str
@@ -41,33 +42,41 @@ class UTMSearchCatalog(StaticCatalogWriterMixin, CatalogSearcher):
     )
     config_cls = UTMSearchConfig
 
-    def __init__(
-        self,
-        endpoint: Optional[MPathLike] = None,
-        collections: List[str] = [],
-        stac_item_modifiers: Optional[List[Callable[[Item], Item]]] = None,
-    ):
-        self.endpoint = endpoint or self.endpoint
-        if len(collections) == 0:  # pragma: no cover
-            raise ValueError("no collections provided")
-        self.collections = collections
-        self.eo_bands = self._eo_bands()
-        self.stac_item_modifiers = stac_item_modifiers
+    @cached_property
+    def eo_bands(self) -> List[str]:  # pragma: no cover
+        for (
+            collection_properties
+        ) in UTMSearchConfig().sinergise_aws_collections.values():
+            if collection_properties["id"] == self.collection.split("/")[-1]:
+                collection = Collection.from_dict(
+                    collection_properties["path"].read_json()
+                )
+                if collection:
+                    summary = collection.summaries.to_dict()
+                    if "eo:bands" in summary:
+                        return summary["eo:bands"]
+                else:
+                    raise ValueError(f"cannot find collection {collection}")
+        else:
+            logger.debug(
+                "cannot find eo:bands definition from collection %s",
+                self.collection,
+            )
+            return []
 
     def search(
         self,
         time: Optional[Union[TimeRange, List[TimeRange]]] = None,
         bounds: Optional[BoundsLike] = None,
         area: Optional[BaseGeometry] = None,
+        query: Optional[str] = None,
         search_kwargs: Optional[Dict[str, Any]] = None,
     ) -> Generator[Item, None, None]:
-        config = self.config_cls(**search_kwargs or {})
-        if bounds:
-            bounds = Bounds.from_inp(bounds)
-
         for item in filter_items(
-            self._raw_search(time=time, bounds=bounds, area=area),
-            max_cloud_cover=config.max_cloud_cover,
+            self._raw_search(
+                time=time, bounds=Bounds.from_inp(bounds) if bounds else None, area=area
+            ),
+            query=query,
         ):
             yield item
 
@@ -88,7 +97,12 @@ class UTMSearchCatalog(StaticCatalogWriterMixin, CatalogSearcher):
         elif bounds is not None:
             bounds = Bounds.from_inp(bounds)
             area = shape(bounds)
-        for time_range in time if isinstance(time, list) else [time]:
+
+        # Cleaner time list in case None present as time (undefined)
+        time_list: list[TimeRange] = (
+            [t for t in time if t is not None] if isinstance(time, list) else [time]
+        )
+        for time_range in time_list:
             start_time = (
                 time_range.start
                 if isinstance(time_range.start, datetime.date)
@@ -147,28 +161,6 @@ class UTMSearchCatalog(StaticCatalogWriterMixin, CatalogSearcher):
                     elif area.intersects(shape(item.geometry)):
                         yield item
 
-    def _eo_bands(self) -> list:
-        for collection_name in self.collections:
-            for (
-                collection_properties
-            ) in UTMSearchConfig().sinergise_aws_collections.values():
-                if collection_properties["id"] == collection_name:
-                    collection = Collection.from_dict(
-                        collection_properties["path"].read_json()
-                    )
-                    if collection:
-                        summary = collection.summaries.to_dict()
-                        if "eo:bands" in summary:
-                            return summary["eo:bands"]
-                    else:
-                        raise ValueError(f"cannot find collection {collection}")
-        else:
-            logger.debug(
-                "cannot find eo:bands definition from collections %s",
-                self.collections,
-            )
-            return []
-
     def get_collections(self):
         """
         yeild transformed collection from:
@@ -178,9 +170,8 @@ class UTMSearchCatalog(StaticCatalogWriterMixin, CatalogSearcher):
         """
         for collection_properties in self.config.sinergise_aws_collections.values():
             collection = Collection.from_dict(collection_properties["path"].read_json())
-            for collection_name in self.collections:
-                if collection_name == collection.id:
-                    yield collection
+            if self.collection.split("/")[-1] == collection.id:
+                yield collection
 
 
 def items_from_static_index(

mapchete_eo/settings.py

@@ -16,6 +16,7 @@ class Settings(BaseSettings):
     default_cache_location: MPathLike = MPath("s3://eox-mhub-cache/")
     default_catalog_crs: CRS = CRS.from_epsg(4326)
     blacklist: Optional[MPathLike] = None
+    lazy_load_stac_items: bool = True
 
     # read from environment
     model_config = SettingsConfigDict(env_prefix="MAPCHETE_EO_")

mapchete_eo/sort.py

@@ -6,7 +6,7 @@ from typing import Callable, List, Optional
 
 from pydantic import BaseModel
 
-from mapchete_eo.protocols import DateTimeProtocol
+from mapchete_eo.protocols import DateTimeProtocol, GetPropertyProtocol
 from mapchete_eo.time import timedelta, to_datetime
 from mapchete_eo.types import DateTimeLike
 
@@ -22,7 +22,7 @@ def sort_objects_by_target_date(
     **kwargs,
 ) -> List[DateTimeProtocol]:
     """
-    Return sorted list of onjects according to their distance to the target_date.
+    Return sorted list of objects according to their distance to the target_date.
 
     Default for target date is the middle between the objects start date and end date.
     """
@@ -46,3 +46,17 @@ class TargetDateSort(SortMethodConfig):
     func: Callable = sort_objects_by_target_date
     target_date: Optional[DateTimeLike] = None
     reverse: bool = False
+
+
+def sort_objects_by_cloud_cover(
+    objects: List[GetPropertyProtocol], reverse: bool = False
+) -> List[GetPropertyProtocol]:
+    if len(objects) == 0:  # pragma: no cover
+        return objects
+    objects.sort(key=lambda x: x.get_property("eo:cloud_cover"), reverse=reverse)
+    return objects
+
+
+class CloudCoverSort(SortMethodConfig):
+    func: Callable = sort_objects_by_cloud_cover
+    reverse: bool = False