mapchete-eo 2025.10.1__py2.py3-none-any.whl → 2026.1.0__py2.py3-none-any.whl

mapchete_eo/search/stac_search.py

@@ -1,95 +1,77 @@
+from __future__ import annotations
+
 import logging
 from datetime import datetime
 from functools import cached_property
-from typing import Any, Callable, Dict, Generator, Iterator, List, Optional, Set, Union
+from typing import Any, Dict, Generator, Iterator, List, Optional, Union
 
 from mapchete import Timer
-from mapchete.path import MPathLike
 from mapchete.tile import BufferedTilePyramid
 from mapchete.types import Bounds, BoundsLike
 from pystac import Item
-from pystac_client import Client
-from shapely.geometry import shape
+from pystac_client import Client, CollectionClient, ItemSearch
+from shapely.geometry import shape, box
 from shapely.geometry.base import BaseGeometry
 
-from mapchete_eo.product import blacklist_products
-from mapchete_eo.search.base import CatalogSearcher, StaticCatalogWriterMixin
-from mapchete_eo.search.config import StacSearchConfig
-from mapchete_eo.settings import mapchete_eo_settings
+from mapchete_eo.search.base import CollectionSearcher, StaticCollectionWriterMixin
+from mapchete_eo.search.config import StacSearchConfig, patch_invalid_assets
 from mapchete_eo.types import TimeRange
 
 logger = logging.getLogger(__name__)
 
 
-class STACSearchCatalog(StaticCatalogWriterMixin, CatalogSearcher):
-    endpoint: str
-    blacklist: Set[str] = (
-        blacklist_products(mapchete_eo_settings.blacklist)
-        if mapchete_eo_settings.blacklist
-        else set()
-    )
+class STACSearchCollection(StaticCollectionWriterMixin, CollectionSearcher):
+    collection: str
     config_cls = StacSearchConfig
 
-    def __init__(
-        self,
-        collections: Optional[List[str]] = None,
-        stac_item_modifiers: Optional[List[Callable[[Item], Item]]] = None,
-        endpoint: Optional[MPathLike] = None,
-    ):
-        if endpoint is not None:
-            self.endpoint = endpoint
-        if collections:
-            self.collections = collections
-        else:  # pragma: no cover
-            raise ValueError("collections must be given")
-        self.stac_item_modifiers = stac_item_modifiers
-
     @cached_property
-    def client(self) -> Client:
-        return Client.open(self.endpoint)
+    def client(self) -> CollectionClient:
+        return CollectionClient.from_file(self.collection)
 
     @cached_property
     def eo_bands(self) -> List[str]:
-        return self._eo_bands()
-
-    @cached_property
-    def id(self) -> str:
-        return self.client.id
-
-    @cached_property
-    def description(self) -> str:
-        return self.client.description
-
-    @cached_property
-    def stac_extensions(self) -> List[str]:
-        return self.client.stac_extensions
+        item_assets = self.client.extra_fields.get("item_assets", {})
+        for v in item_assets.values():
+            if "eo:bands" in v and "data" in v.get("roles", []):
+                return ["eo:bands"]
+        else:  # pragma: no cover
+            logger.debug("cannot find eo:bands definition from collections")
+            return []
 
     def search(
         self,
         time: Optional[Union[TimeRange, List[TimeRange]]] = None,
         bounds: Optional[BoundsLike] = None,
         area: Optional[BaseGeometry] = None,
+        query: Optional[str] = None,
         search_kwargs: Optional[Dict[str, Any]] = None,
     ) -> Generator[Item, None, None]:
         config = self.config_cls(**search_kwargs or {})
         if bounds:
             bounds = Bounds.from_inp(bounds)
-        if time is None:  # pragma: no cover
-            raise ValueError("time must be set")
         if area is None and bounds is None:  # pragma: no cover
             raise ValueError("either bounds or area have to be given")
 
         if area is not None and area.is_empty:  # pragma: no cover
             return
 
-        def _searches():
-            for time_range in time if isinstance(time, list) else [time]:
+        def _searches() -> Generator[ItemSearch, None, None]:
+            def _search_chunks(
+                time_range: Optional[TimeRange] = None,
+                bounds: Optional[BoundsLike] = None,
+                area: Optional[BaseGeometry] = None,
+                query: Optional[str] = None,
+            ):
                 search = self._search(
-                    time_range=time_range, bounds=bounds, area=area, config=config
+                    time_range=time_range,
+                    bounds=bounds,
+                    area=box(*area.bounds) if area else None,
+                    query=query,
+                    config=config,
                 )
                 logger.debug("found %s products", search.matched())
                 matched = search.matched() or 0
-                if matched > config.catalog_chunk_threshold:
+                if matched > config.catalog_chunk_threshold:  # pragma: no cover
                     spatial_search_chunks = SpatialSearchChunks(
                         bounds=bounds,
                         area=area,
@@ -105,6 +87,7 @@ class STACSearchCatalog(StaticCatalogWriterMixin, CatalogSearcher):
                         with Timer() as duration:
                             chunk_search = self._search(
                                 time_range=time_range,
+                                query=query,
                                 config=config,
                                 **chunk_kwargs,
                             )
@@ -119,48 +102,62 @@ class STACSearchCatalog(StaticCatalogWriterMixin, CatalogSearcher):
                 else:
                     yield search
 
-        for search in _searches():
-            for count, item in enumerate(search.items(), 1):
-                item_path = item.get_self_href()
-                # logger.debug("item %s/%s ...", count, search.matched())
-                if item_path in self.blacklist:  # pragma: no cover
-                    logger.debug("item %s found in blacklist and skipping", item_path)
-                else:
-                    yield item
+            if time:
+                # search time range(s)
+                for time_range in time if isinstance(time, list) else [time]:
+                    yield from _search_chunks(
+                        time_range=time_range,
+                        bounds=bounds,
+                        area=area,
+                        query=query,
+                    )
+            else:
+                # don't apply temporal filter
+                yield from _search_chunks(
+                    bounds=bounds,
+                    area=area,
+                    query=query,
+                )
 
-    def _eo_bands(self) -> List[str]:
-        for collection_name in self.collections:
-            collection = self.client.get_collection(collection_name)
-            if collection:
-                item_assets = collection.extra_fields.get("item_assets", {})
-                for v in item_assets.values():
-                    if "eo:bands" in v and "data" in v.get("roles", []):
-                        return ["eo:bands"]
-            else:  # pragma: no cover
-                raise ValueError(f"cannot find collection {collection}")
-        else:  # pragma: no cover
-            logger.debug("cannot find eo:bands definition from collections")
-            return []
+        with patch_invalid_assets():
+            for search in _searches():
+                for item in search.items():
+                    if item.get_self_href() in self.blacklist:  # pragma: no cover
+                        logger.debug(
+                            "item %s found in blacklist and skipping",
+                            item.get_self_href(),
+                        )
+                        continue
+                    yield item
 
     @cached_property
     def default_search_params(self):
         return {
-            "collections": self.collections,
+            "collections": [self.client],
             "bbox": None,
             "intersects": None,
         }
 
+    @cached_property
+    def search_client(self) -> Client:
+        # looks weird, right?
+        #
+        # one would assume that directly returning self.client.get_root() would
+        # do the same but if we do so, it seems to ignore the "collections" parameter
+        # and thus query all collection available on that search endpoint.
+        #
+        # the only way to fix this, is to instantiate Client from scratch.
+        return Client.from_file(self.client.get_root().self_href)
+
     def _search(
         self,
         time_range: Optional[TimeRange] = None,
         bounds: Optional[Bounds] = None,
         area: Optional[BaseGeometry] = None,
+        query: Optional[str] = None,
         config: StacSearchConfig = StacSearchConfig(),
         **kwargs,
-    ):
-        if time_range is None:  # pragma: no cover
-            raise ValueError("time_range not provided")
-
+    ) -> ItemSearch:
         if bounds is not None:
             if shape(bounds).is_empty:  # pragma: no cover
                 raise ValueError("bounds empty")
@@ -170,22 +167,29 @@ class STACSearchCatalog(StaticCatalogWriterMixin, CatalogSearcher):
                 raise ValueError("area empty")
             kwargs.update(intersects=area)
 
-        start = (
-            time_range.start.date()
-            if isinstance(time_range.start, datetime)
-            else time_range.start
-        )
-        end = (
-            time_range.end.date()
-            if isinstance(time_range.end, datetime)
-            else time_range.end
-        )
-        search_params = dict(
-            self.default_search_params,
-            datetime=f"{start}/{end}",
-            query=[f"eo:cloud_cover<={config.max_cloud_cover}"],
-            **kwargs,
-        )
+        if time_range:
+            start = (
+                time_range.start.date()
+                if isinstance(time_range.start, datetime)
+                else time_range.start
+            )
+            end = (
+                time_range.end.date()
+                if isinstance(time_range.end, datetime)
+                else time_range.end
+            )
+            search_params = dict(
+                self.default_search_params,
+                datetime=f"{start}/{end}",
+                query=[query] if query else None,
+                **kwargs,
+            )
+        else:
+            search_params = dict(
+                self.default_search_params,
+                query=[query] if query else None,
+                **kwargs,
+            )
         if (
             bounds is None
             and area is None
@@ -194,14 +198,12 @@ class STACSearchCatalog(StaticCatalogWriterMixin, CatalogSearcher):
             raise ValueError("no bounds or area given")
         logger.debug("query catalog using params: %s", search_params)
         with Timer() as duration:
-            result = self.client.search(**search_params, limit=config.catalog_pagesize)
+            result = self.search_client.search(
+                **search_params, limit=config.catalog_pagesize
+            )
         logger.debug("query took %s", str(duration))
         return result
 
-    def get_collections(self):
-        for collection_name in self.collections:
-            yield self.client.get_collection(collection_name)
-
 
 class SpatialSearchChunks:
     bounds: Bounds

mapchete_eo/search/stac_static.py

@@ -1,22 +1,21 @@
 from functools import cached_property
 import logging
 import warnings
-from typing import Any, Callable, Dict, Generator, List, Optional, Union
+from typing import Any, Dict, Generator, List, Optional, Union
 
 from mapchete import Bounds
 from mapchete.types import BoundsLike
 from pystac import Item, Catalog, Collection
 from mapchete.io.vector import bounds_intersect
-from mapchete.path import MPathLike
 from pystac.stac_io import StacIO
-from pystac_client import Client
+from pystac_client import CollectionClient
 from shapely.geometry import shape
 from shapely.geometry.base import BaseGeometry
 
 from mapchete_eo.search.base import (
-    CatalogSearcher,
+    CollectionSearcher,
     FSSpecStacIO,
-    StaticCatalogWriterMixin,
+    StaticCollectionWriterMixin,
     filter_items,
 )
 from mapchete_eo.search.config import StacStaticConfig
@@ -29,49 +28,55 @@ logger = logging.getLogger(__name__)
 StacIO.set_default(FSSpecStacIO)
 
 
-class STACStaticCatalog(StaticCatalogWriterMixin, CatalogSearcher):
+class STACStaticCollection(StaticCollectionWriterMixin, CollectionSearcher):
     config_cls = StacStaticConfig
 
-    def __init__(
-        self,
-        baseurl: MPathLike,
-        stac_item_modifiers: Optional[List[Callable[[Item], Item]]] = None,
-    ):
-        self.client = Client.from_file(str(baseurl), stac_io=FSSpecStacIO())
-        self.collections = [c.id for c in self.client.get_children()]
-        self.stac_item_modifiers = stac_item_modifiers
-
-    @cached_property
-    def eo_bands(self) -> List[str]:
-        return self._eo_bands()
-
     @cached_property
-    def id(self) -> str:
-        return self.client.id
+    def client(self) -> CollectionClient:
+        return CollectionClient.from_file(str(self.collection), stac_io=FSSpecStacIO())
 
     @cached_property
-    def description(self) -> str:
-        return self.client.description
+    def eo_bands(self) -> List[str]:
+        eo_bands = self.client.extra_fields.get("properties", {}).get("eo:bands")
+        if eo_bands:
+            return eo_bands
+        else:
+            warnings.warn(
+                "Unable to read eo:bands definition from collection. "
+                "Trying now to get information from assets ..."
+            )
+            # see if eo:bands can be found in properties
+            try:
+                item = next(self.client.get_items(recursive=True))
+                eo_bands = item.properties.get("eo:bands")
+                if eo_bands:
+                    return eo_bands
+
+                # look through the assets and collect eo:bands
+                out = {}
+                for asset in item.assets.values():
+                    for eo_band in asset.extra_fields.get("eo:bands", []):
+                        out[eo_band["name"]] = eo_band
+                if out:
+                    return [v for v in out.values()]
+            except StopIteration:
+                pass
 
-    @cached_property
-    def stac_extensions(self) -> List[str]:
-        return self.client.stac_extensions
+            logger.debug("cannot find eo:bands definition")
+            return []
 
     def search(
         self,
         time: Optional[Union[TimeRange, List[TimeRange]]] = None,
         bounds: Optional[BoundsLike] = None,
         area: Optional[BaseGeometry] = None,
+        query: Optional[str] = None,
         search_kwargs: Optional[Dict[str, Any]] = None,
     ) -> Generator[Item, None, None]:
-        config = self.config_cls(**search_kwargs or {})
         if area is None and bounds:
             bounds = Bounds.from_inp(bounds)
             area = shape(bounds)
-        for item in filter_items(
-            self._raw_search(time=time, area=area),
-            max_cloud_cover=config.max_cloud_cover,
-        ):
+        for item in filter_items(self._raw_search(time=time, area=area), query=query):
             yield item
 
     def _raw_search(
@@ -82,83 +87,22 @@ class STACStaticCatalog(StaticCatalogWriterMixin, CatalogSearcher):
         if area is not None and area.is_empty:
             return
         logger.debug("iterate through children")
-        for collection in self.client.get_collections():
-            if time:
-                for time_range in time if isinstance(time, list) else [time]:
-                    for item in _all_intersecting_items(
-                        collection,
-                        area=area,
-                        time_range=time_range,
-                    ):
-                        item.make_asset_hrefs_absolute()
-                        yield item
-            else:
+        if time:
+            for time_range in time if isinstance(time, list) else [time]:
                 for item in _all_intersecting_items(
-                    collection,
+                    self.client,
                     area=area,
+                    time_range=time_range,
                 ):
                     item.make_asset_hrefs_absolute()
                     yield item
-
-    def _eo_bands(self) -> List[str]:
-        for collection in self.client.get_children():
-            eo_bands = collection.extra_fields.get("properties", {}).get("eo:bands")
-            if eo_bands:
-                return eo_bands
         else:
-            warnings.warn(
-                "Unable to read eo:bands definition from collections. "
-                "Trying now to get information from assets ..."
-            )
-
-            # see if eo:bands can be found in properties
-            item = _get_first_item(self.client.get_children())
-            eo_bands = item.properties.get("eo:bands")
-            if eo_bands:
-                return eo_bands
-
-            # look through the assets and collect eo:bands
-            out = {}
-            for asset in item.assets.values():
-                for eo_band in asset.extra_fields.get("eo:bands", []):
-                    out[eo_band["name"]] = eo_band
-            if out:
-                return [v for v in out.values()]
-
-            logger.debug("cannot find eo:bands definition")
-            return []
-
-    def get_collections(
-        self,
-        time: Optional[Union[TimeRange, List[TimeRange]]] = None,
-        bounds: Optional[BoundsLike] = None,
-        area: Optional[BaseGeometry] = None,
-    ):
-        if area is None and bounds is not None:
-            area = Bounds.from_inp(bounds).geometry
-        for collection in self.client.get_children():
-            if time:
-                for time_range in time if isinstance(time, list) else [time]:
-                    if _collection_extent_intersects(
-                        collection,
-                        area=area,
-                        time_range=time_range,
-                    ):
-                        yield collection
-            else:
-                if _collection_extent_intersects(collection, area=area):
-                    yield collection
-
-
-def _get_first_item(collections):
-    for collection in collections:
-        for item in collection.get_all_items():
-            return item
-        else:
-            for child in collection.get_children():
-                return _get_first_item(child)
-    else:
-        raise ValueError("collections contain no items")
+            for item in _all_intersecting_items(
+                self.client,
+                area=area,
+            ):
+                item.make_asset_hrefs_absolute()
+                yield item
 
 
 def _all_intersecting_items(

mapchete_eo/search/utm_search.py

@@ -1,77 +1,86 @@
 import datetime
 from functools import cached_property
 import logging
-from typing import Any, Callable, Dict, Generator, List, Optional, Set, Union
+from typing import Any, Dict, Generator, List, Optional, Union
 
 from mapchete.io.vector import fiona_open
 from mapchete.path import MPath, MPathLike
 from mapchete.types import Bounds, BoundsLike
 from pystac.collection import Collection
 from pystac.item import Item
+from pystac_client import CollectionClient
 from shapely.errors import GEOSException
 from shapely.geometry import shape
 from shapely.geometry.base import BaseGeometry
 
 from mapchete_eo.exceptions import ItemGeometryError
-from mapchete_eo.product import blacklist_products
 from mapchete_eo.search.base import (
-    CatalogSearcher,
-    StaticCatalogWriterMixin,
+    CollectionSearcher,
+    StaticCollectionWriterMixin,
     filter_items,
 )
 from mapchete_eo.search.config import UTMSearchConfig
 from mapchete_eo.search.s2_mgrs import S2Tile, s2_tiles_from_bounds
-from mapchete_eo.settings import mapchete_eo_settings
 from mapchete_eo.time import day_range, to_datetime
 from mapchete_eo.types import TimeRange
 
 logger = logging.getLogger(__name__)
 
 
-class UTMSearchCatalog(StaticCatalogWriterMixin, CatalogSearcher):
-    endpoint: str
-    id: str
-    day_subdir_schema: str
-    stac_json_endswith: str
-    description: str
-    stac_extensions: List[str]
-    blacklist: Set[str] = (
-        blacklist_products(mapchete_eo_settings.blacklist)
-        if mapchete_eo_settings.blacklist
-        else set()
-    )
+class UTMSearchCatalog(StaticCollectionWriterMixin, CollectionSearcher):
     config_cls = UTMSearchConfig
 
-    def __init__(
-        self,
-        endpoint: Optional[MPathLike] = None,
-        collections: List[str] = [],
-        stac_item_modifiers: Optional[List[Callable[[Item], Item]]] = None,
-    ):
-        self.endpoint = endpoint or self.endpoint
-        if len(collections) == 0:  # pragma: no cover
-            raise ValueError("no collections provided")
-        self.collections = collections
-        self.stac_item_modifiers = stac_item_modifiers
+    @cached_property
+    def endpoint(self) -> Optional[str]:
+        for collection_properties in self.config.sinergise_aws_collections.values():
+            if collection_properties["id"] == self.collection.split("/")[-1].replace(
+                ".json", ""
+            ):
+                return collection_properties.get("endpoint")
+        return None
+
+    day_subdir_schema: str = "{year}/{month:02d}/{day:02d}"
+    stac_json_endswith: str = "T{tile_id}.json"
+
+    @cached_property
+    def client(self) -> CollectionClient:
+        return next(self.get_collections())
 
     @cached_property
     def eo_bands(self) -> List[str]:  # pragma: no cover
-        return self._eo_bands()
+        for (
+            collection_properties
+        ) in UTMSearchConfig().sinergise_aws_collections.values():
+            if collection_properties["id"] == self.collection.split("/")[-1]:
+                collection = Collection.from_dict(
+                    collection_properties["path"].read_json()
+                )
+                if collection:
+                    summary = collection.summaries.to_dict()
+                    if "eo:bands" in summary:
+                        return summary["eo:bands"]
+                else:
+                    raise ValueError(f"cannot find collection {collection}")
+        else:
+            logger.debug(
+                "cannot find eo:bands definition from collection %s",
+                self.collection,
+            )
+            return []
 
     def search(
         self,
         time: Optional[Union[TimeRange, List[TimeRange]]] = None,
         bounds: Optional[BoundsLike] = None,
         area: Optional[BaseGeometry] = None,
+        query: Optional[str] = None,
         search_kwargs: Optional[Dict[str, Any]] = None,
     ) -> Generator[Item, None, None]:
-        config = self.config_cls(**search_kwargs or {})
-        if bounds:
-            bounds = Bounds.from_inp(bounds)
-
         for item in filter_items(
-            self._raw_search(time=time, bounds=bounds, area=area),
-            max_cloud_cover=config.max_cloud_cover,
+            self._raw_search(
+                time=time, bounds=Bounds.from_inp(bounds) if bounds else None, area=area
+            ),
+            query=query,
         ):
             yield item
 
@@ -80,8 +89,9 @@ class UTMSearchCatalog(StaticCatalogWriterMixin, CatalogSearcher):
         time: Optional[Union[TimeRange, List[TimeRange]]] = None,
         bounds: Optional[Bounds] = None,
         area: Optional[BaseGeometry] = None,
-        config: UTMSearchConfig = UTMSearchConfig(),
+        config: Optional[UTMSearchConfig] = None,
     ) -> Generator[Item, None, None]:
+        config = config or UTMSearchConfig()
         if time is None:
             raise ValueError("time must be given")
         if area is not None and area.is_empty:
@@ -92,7 +102,12 @@ class UTMSearchCatalog(StaticCatalogWriterMixin, CatalogSearcher):
         elif bounds is not None:
             bounds = Bounds.from_inp(bounds)
             area = shape(bounds)
-        for time_range in time if isinstance(time, list) else [time]:
+
+        # Cleaner time list in case None present as time (undefined)
+        time_list: list[TimeRange] = (
+            [t for t in time if t is not None] if isinstance(time, list) else [time]
+        )
+        for time_range in time_list:
             start_time = (
                 time_range.start
                 if isinstance(time_range.start, datetime.date)
@@ -151,28 +166,6 @@ class UTMSearchCatalog(StaticCatalogWriterMixin, CatalogSearcher):
                     elif area.intersects(shape(item.geometry)):
                         yield item
 
-    def _eo_bands(self) -> list:
-        for collection_name in self.collections:
-            for (
-                collection_properties
-            ) in UTMSearchConfig().sinergise_aws_collections.values():
-                if collection_properties["id"] == collection_name:
-                    collection = Collection.from_dict(
-                        collection_properties["path"].read_json()
-                    )
-                    if collection:
-                        summary = collection.summaries.to_dict()
-                        if "eo:bands" in summary:
-                            return summary["eo:bands"]
-                    else:
-                        raise ValueError(f"cannot find collection {collection}")
-        else:
-            logger.debug(
-                "cannot find eo:bands definition from collections %s",
-                self.collections,
-            )
-            return []
-
     def get_collections(self):
         """
         yeild transformed collection from:
@@ -182,9 +175,8 @@ class UTMSearchCatalog(StaticCatalogWriterMixin, CatalogSearcher):
         """
         for collection_properties in self.config.sinergise_aws_collections.values():
             collection = Collection.from_dict(collection_properties["path"].read_json())
-            for collection_name in self.collections:
-                if collection_name == collection.id:
-                    yield collection
+            if self.collection.split("/")[-1] == collection.id:
+                yield collection
 
 
 def items_from_static_index(