eodag 3.10.1__py3-none-any.whl → 4.0.0a2__py3-none-any.whl

eodag/plugins/search/cop_marine.py

@@ -34,11 +34,12 @@ from dateutil.utils import today
 
 from eodag import EOProduct
 from eodag.api.product import AssetsDict
+from eodag.api.search_result import SearchResult
 from eodag.config import PluginConfig
 from eodag.plugins.search import PreparedSearch
 from eodag.plugins.search.static_stac_search import StaticStacSearch
 from eodag.utils import get_bucket_name_and_prefix, get_geometry_from_various
-from eodag.utils.exceptions import RequestError, UnsupportedProductType, ValidationError
+from eodag.utils.exceptions import RequestError, UnsupportedCollection, ValidationError
 
 if TYPE_CHECKING:
     from mypy_boto3_s3 import S3Client
@@ -112,7 +113,7 @@ def _check_int_values_properties(properties: dict[str, Any]):
 class CopMarineSearch(StaticStacSearch):
     """class that implements search for the Copernicus Marine provider
 
-    It calls :meth:`~eodag.plugins.search.static_stac_search.StaticStacSearch.discover_product_types`
+    It calls :meth:`~eodag.plugins.search.static_stac_search.StaticStacSearch.discover_collections`
     inherited from :class:`~eodag.plugins.search.static_stac_search.StaticStacSearch`
     but for the actual search a special method which fetches the urls of the available products from an S3 storage and
     filters them has been written.
@@ -132,40 +133,40 @@ class CopMarineSearch(StaticStacSearch):
         # reset to original metadata mapping from config (changed in super class init)
         self.config.metadata_mapping = original_metadata_mapping
 
-    def _get_product_type_info(
-        self, product_type: str
+    def _get_collection_info(
+        self, collection: str
     ) -> tuple[dict[str, Any], list[dict[str, Any]]]:
-        """Fetch product type and associated datasets info"""
+        """Fetch collection and associated datasets info"""
 
-        fetch_url = cast(str, self.config.discover_product_types["fetch_url"]).format(
+        fetch_url = cast(str, self.config.discover_collections["fetch_url"]).format(
             **self.config.__dict__
         )
 
-        logger.debug("fetch data for collection %s", product_type)
-        provider_product_type = self.config.products.get(product_type, {}).get(
-            "productType", None
+        logger.debug("fetch data for collection %s", collection)
+        provider_collection = self.config.products.get(collection, {}).get(
+            "_collection", None
         )
-        if not provider_product_type:
-            provider_product_type = product_type
+        if not provider_collection:
+            provider_collection = collection
         collection_url = (
-            fetch_url.replace("catalog.stac.json", provider_product_type)
+            fetch_url.replace("catalog.stac.json", provider_collection)
             + "/product.stac.json"
         )
         try:
             collection_data = requests.get(collection_url).json()
         except requests.RequestException as exc:
             if exc.errno == 404:
-                logger.error("product %s not found", product_type)
-                raise UnsupportedProductType(product_type)
-            logger.error("data for product %s could not be fetched", product_type)
+                logger.error("product %s not found", collection)
+                raise UnsupportedCollection(collection)
+            logger.error("data for product %s could not be fetched", collection)
             raise RequestError.from_error(
-                exc, f"data for product {product_type} could not be fetched"
+                exc, f"data for product {collection} could not be fetched"
             ) from exc
 
         datasets = []
         for link in [li for li in collection_data["links"] if li["rel"] == "item"]:
             dataset_url = (
-                fetch_url.replace("catalog.stac.json", provider_product_type)
+                fetch_url.replace("catalog.stac.json", provider_collection)
                 + "/"
                 + link["href"]
             )
@@ -182,7 +183,7 @@ class CopMarineSearch(StaticStacSearch):
         collection_objects: ListObjectsOutputTypeDef,
         product_id: str,
         s3_url: str,
-        product_type: str,
+        collection: str,
         dataset_item: dict[str, Any],
         collection_dict: dict[str, Any],
     ):
@@ -194,7 +195,7 @@ class CopMarineSearch(StaticStacSearch):
         for obj in collection_objects["Contents"]:
             if product_id in obj["Key"]:
                 return self._create_product(
-                    product_type,
+                    collection,
                     obj["Key"],
                     s3_url,
                     dataset_item,
@@ -205,7 +206,7 @@ class CopMarineSearch(StaticStacSearch):
 
     def _create_product(
         self,
-        product_type: str,
+        collection: str,
         item_key: str,
         s3_url: str,
         dataset_item: dict[str, Any],
@@ -220,21 +221,21 @@ class CopMarineSearch(StaticStacSearch):
         download_url = s3_url + "/" + item_key
         geometry = (
             get_geometry_from_various(**dataset_item)
-            or self.config.metadata_mapping["defaultGeometry"]
+            or self.config.metadata_mapping["eodag:default_geometry"]
         )
         properties = {
             "id": item_id,
             "title": item_id,
             "geometry": geometry,
-            "downloadLink": download_url,
+            "eodag:download_link": download_url,
             "dataset": dataset_item["id"],
         }
         if use_dataset_dates:
             dates = _get_dates_from_dataset_data(dataset_item)
             if not dates:
                 return None
-            properties["startTimeFromAscendingNode"] = dates["start"]
-            properties["completionTimeFromAscendingNode"] = dates["end"]
+            properties["start_datetime"] = dates["start"]
+            properties["end_datetime"] = dates["end"]
         else:
             item_dates = re.findall(r"(\d{4})(0[1-9]|1[0-2])([0-3]\d)", item_id)
             if not item_dates:
@@ -247,12 +248,10 @@ class CopMarineSearch(StaticStacSearch):
                 item_end = _get_date_from_yyyymmdd(item_dates[1], item_key)
             else:  # only date and created_at timestamps
                 item_end = item_start
-            properties["startTimeFromAscendingNode"] = item_start.strftime(
+            properties["start_datetime"] = item_start.strftime("%Y-%m-%dT%H:%M:%SZ")
+            properties["end_datetime"] = (item_end or item_start).strftime(
                 "%Y-%m-%dT%H:%M:%SZ"
             )
-            properties["completionTimeFromAscendingNode"] = (
-                item_end or item_start
-            ).strftime("%Y-%m-%dT%H:%M:%SZ")
 
         for key, value in collection_dict["properties"].items():
             if key not in ["id", "title", "start_datetime", "end_datetime", "datetime"]:
@@ -261,7 +260,7 @@ class CopMarineSearch(StaticStacSearch):
             if key not in ["id", "title", "start_datetime", "end_datetime", "datetime"]:
                 properties[key] = value
 
-        code_mapping = self.config.products.get(product_type, {}).get(
+        code_mapping = self.config.products.get(collection, {}).get(
             "code_mapping", None
         )
         if code_mapping:
@@ -277,9 +276,11 @@ class CopMarineSearch(StaticStacSearch):
 
         _check_int_values_properties(properties)
 
-        properties["thumbnail"] = collection_dict["assets"]["thumbnail"]["href"]
+        properties["eodag:thumbnail"] = collection_dict["assets"]["thumbnail"]["href"]
         if "omiFigure" in collection_dict["assets"]:
-            properties["quicklook"] = collection_dict["assets"]["omiFigure"]["href"]
+            properties["eodag:quicklook"] = collection_dict["assets"]["omiFigure"][
+                "href"
+            ]
         assets = {
             "native": {
                 "title": "native",
@@ -289,10 +290,7 @@ class CopMarineSearch(StaticStacSearch):
         }
         additional_assets = self.get_assets_from_mapping(dataset_item)
         assets.update(additional_assets)
-        product = EOProduct(self.provider, properties, productType=product_type)
-        # use product_type_config as default properties
-        product_type_config = getattr(self.config, "product_type_config", {})
-        product.properties = dict(product_type_config, **product.properties)
+        product = EOProduct(self.provider, properties, collection=collection)
         product.assets = AssetsDict(product, assets)
         return product
 
@@ -300,29 +298,39 @@ class CopMarineSearch(StaticStacSearch):
         self,
         prep: PreparedSearch = PreparedSearch(),
         **kwargs: Any,
-    ) -> tuple[list[EOProduct], Optional[int]]:
+    ) -> SearchResult:
         """
         Implementation of search for the Copernicus Marine provider
         :param prep: object containing search parameterds
         :param kwargs: additional search arguments
         :returns: list of products and total number of products
         """
-        page = prep.page
         items_per_page = prep.items_per_page
+        token_value = getattr(prep, "next_page_token") or prep.page
 
         # only return 1 page if pagination is disabled
-        if page is None or items_per_page is None or page > 1 and items_per_page <= 0:
-            return ([], 0) if prep.count else ([], None)
-
-        product_type = kwargs.get("productType", prep.product_type)
-        if not product_type:
+        if (
+            token_value is None
+            or items_per_page is None
+            or int(token_value) > 1
+            and items_per_page <= 0
+        ):
+            result = SearchResult([])
+            if prep.count:
+                result.number_matched = 0
+            return result
+
+        token = int(token_value)
+
+        collection = kwargs.get("collection", prep.collection)
+        if not collection:
             raise ValidationError(
-                "parameter product type is required for search with cop_marine provider"
+                "parameter collection is required for search with cop_marine provider"
             )
-        collection_dict, datasets_items_list = self._get_product_type_info(product_type)
+        collection_dict, datasets_items_list = self._get_collection_info(collection)
         geometry = kwargs.pop("geometry", None)
         products: list[EOProduct] = []
-        start_index = items_per_page * (page - 1) + 1
+        start_index = items_per_page * (token - 1) + 1
         num_total = 0
         for i, dataset_item in enumerate(datasets_items_list):
             # Filter by geometry
@@ -334,16 +342,16 @@ class CopMarineSearch(StaticStacSearch):
                 logger.debug("searching data for dataset %s", dataset_item["id"])
 
                 # date bounds
-                if "startTimeFromAscendingNode" in kwargs:
-                    start_date = isoparse(kwargs["startTimeFromAscendingNode"])
+                if "start_datetime" in kwargs:
+                    start_date = isoparse(kwargs["start_datetime"])
                 elif "start_datetime" in dataset_item["properties"]:
                     start_date = isoparse(dataset_item["properties"]["start_datetime"])
                 else:
                     start_date = isoparse(dataset_item["properties"]["datetime"])
                 if not start_date.tzinfo:
                     start_date = start_date.replace(tzinfo=tzutc())
-                if "completionTimeFromAscendingNode" in kwargs:
-                    end_date = isoparse(kwargs["completionTimeFromAscendingNode"])
+                if "end_datetime" in kwargs:
+                    end_date = isoparse(kwargs["end_datetime"])
                 elif "end_datetime" in dataset_item["properties"]:
                     end_date = isoparse(dataset_item["properties"]["end_datetime"])
                 else:
@@ -355,7 +363,7 @@ class CopMarineSearch(StaticStacSearch):
                 s3_url = dataset_item["assets"]["native"]["href"]
             except KeyError as e:
                 logger.warning(
-                    f"Unable to extract info from {product_type} item #{i}: {str(e)}"
+                    f"Unable to extract info from {collection} item #{i}: {str(e)}"
                 )
                 continue
 
@@ -374,7 +382,7 @@ class CopMarineSearch(StaticStacSearch):
                     continue
                 if len(products) < items_per_page or items_per_page < 0:
                     product = self._create_product(
-                        product_type,
+                        collection,
                         collection_path,
                         endpoint_url + "/" + bucket,
                         dataset_item,
@@ -403,7 +411,10 @@ class CopMarineSearch(StaticStacSearch):
                     )
                 if "Contents" not in s3_objects:
                     if len(products) == 0 and i == len(datasets_items_list) - 1:
-                        return ([], 0) if prep.count else ([], None)
+                        result = SearchResult([])
+                        if prep.count:
+                            result.number_matched = 0
+                        return result
                     else:
                         break
 
@@ -412,12 +423,16 @@ class CopMarineSearch(StaticStacSearch):
                         s3_objects,
                         kwargs["id"],
                         endpoint_url + "/" + bucket,
-                        product_type,
+                        collection,
                         dataset_item,
                         collection_dict,
                     )
                     if product:
-                        return [product], 1
+                        formated_result = SearchResult(
+                            [product],
+                            1,
+                        )
+                        return formated_result
                     current_object = s3_objects["Contents"][-1]["Key"]
                     continue
 
@@ -476,7 +491,7 @@ class CopMarineSearch(StaticStacSearch):
                             continue
                         if len(products) < items_per_page or items_per_page < 0:
                             product = self._create_product(
-                                product_type,
+                                collection,
                                 item_key,
                                 endpoint_url + "/" + bucket,
                                 dataset_item,
@@ -487,4 +502,20 @@ class CopMarineSearch(StaticStacSearch):
                                 products.append(product)
                     current_object = item_key
 
-        return products, num_total
+        search_params = (
+            kwargs
+            | {"items_per_page": prep.items_per_page}
+            | {"collection": collection}
+            | {"provider": self.provider}
+            | {"geometry": geometry}
+            if geometry
+            else {}
+        )
+
+        formated_result = SearchResult(
+            products,
+            num_total,
+            search_params=search_params,
+            next_page_token=str(start_index + 1),
+        )
+        return formated_result

eodag/plugins/search/csw.py

@@ -35,6 +35,7 @@ from shapely import geometry, wkt
 
 from eodag.api.product import EOProduct
 from eodag.api.product.metadata_mapping import properties_from_xml
+from eodag.api.search_result import SearchResult
 from eodag.plugins.search import PreparedSearch
 from eodag.plugins.search.base import Search
 from eodag.utils import DEFAULT_PROJ
@@ -62,7 +63,7 @@ class CSWSearch(Search):
         * :attr:`~eodag.config.PluginConfig.version` (``str``): OGC Catalogue Service version; default: ``2.0.2``
         * :attr:`~eodag.config.PluginConfig.search_definition` (``dict[str, Any]``) (**mandatory**):
 
-          * **product_type_tags** (``list[dict[str, Any]``): dict of product type tags
+          * **collection_tags** (``list[dict[str, Any]``): dict of collection tags
           * **resource_location_filter** (``str``): regex string
           * **date_tags** (``dict[str, Any]``): tags for start and end
 
@@ -74,15 +75,15 @@ class CSWSearch(Search):
           specification of Python string formatting, with a special behaviour added to it. For example,
           an entry in the metadata mapping of this kind::
 
-                completionTimeFromAscendingNode:
-                    - 'f=acquisition.endViewingDate:lte:{completionTimeFromAscendingNode#timestamp}'
+                end_datetime:
+                    - 'f=acquisition.endViewingDate:lte:{end_datetime#timestamp}'
                     - '$.properties.acquisition.endViewingDate'
 
           means that the search url will have a query string parameter named ``f`` with a value of
           ``acquisition.endViewingDate:lte:1543922280.0`` if the search was done with the value
-          of ``completionTimeFromAscendingNode`` being ``2018-12-04T12:18:00``. What happened is that
-          ``{completionTimeFromAscendingNode#timestamp}`` was replaced with the timestamp of the value
-          of ``completionTimeFromAscendingNode``. This example shows all there is to know about the
+          of ``end_datetime`` being ``2018-12-04T12:18:00``. What happened is that
+          ``{end_datetime#timestamp}`` was replaced with the timestamp of the value
+          of ``end_datetime``. This example shows all there is to know about the
           semantics of the query string formatting introduced by this plugin: any eodag search parameter
           can be referenced in the query string with an additional optional conversion function that
           is separated from it by a ``#`` (see :func:`~eodag.api.product.metadata_mapping.format_metadata` for further
@@ -107,11 +108,14 @@ class CSWSearch(Search):
         self,
         prep: PreparedSearch = PreparedSearch(),
         **kwargs: Any,
-    ) -> tuple[list[EOProduct], Optional[int]]:
+    ) -> SearchResult:
         """Perform a search on a OGC/CSW-like interface"""
-        product_type = kwargs.get("productType")
-        if product_type is None:
-            return ([], 0) if prep.count else ([], None)
+        collection = kwargs.get("collection")
+        if collection is None:
+            result = SearchResult([])
+            if prep.count:
+                result.number_matched = 0
+            return result
         auth = kwargs.get("auth")
         if auth:
             self.__init_catalog(**getattr(auth.config, "credentials", {}))
@@ -119,16 +123,16 @@ class CSWSearch(Search):
             self.__init_catalog()
         results: list[EOProduct] = []
         if self.catalog:
-            provider_product_type = self.config.products[product_type]["productType"]
-            for product_type_def in self.config.search_definition["product_type_tags"]:
-                product_type_search_tag = product_type_def["name"]
+            provider_collection = self.config.products[collection]["collection"]
+            for collection_def in self.config.search_definition["collection_tags"]:
+                collection_search_tag = collection_def["name"]
                 logger.debug(
-                    "Querying <%s> tag for product type %s",
-                    product_type_search_tag,
-                    provider_product_type,
+                    "Querying <%s> tag for collection %s",
+                    collection_search_tag,
+                    provider_collection,
                 )
                 constraints = self.__convert_query_params(
-                    product_type_def, provider_product_type, kwargs
+                    collection_def, provider_collection, kwargs
                 )
                 with patch_owslib_requests(verify=True):
                     try:
@@ -139,25 +143,31 @@ class CSWSearch(Search):
                         import traceback as tb
 
                         logger.warning(
-                            "Failed to query %s for product type %s : %s",
-                            product_type_search_tag,
-                            product_type,
+                            "Failed to query %s for collection %s : %s",
+                            collection_search_tag,
+                            collection,
                             tb.format_exc(),
                         )
                         continue
                 partial_results = [
-                    self.__build_product(record, product_type, **kwargs)
+                    self.__build_product(record, collection, **kwargs)
                     for record in self.catalog.records.values()
                 ]
                 logger.info(
                     "Found %s results querying %s",
                     len(partial_results),
-                    product_type_search_tag,
+                    collection_search_tag,
                 )
                 results.extend(partial_results)
         logger.info("Found %s overall results", len(results))
         total_results = len(results) if prep.count else None
-        return results, total_results
+        if not prep.count and "number_matched" in kwargs:
+            total_results = kwargs["number_matched"]
+        formated_result = SearchResult(
+            results,
+            total_results,
+        )
+        return formated_result
 
     def __init_catalog(
         self, username: Optional[str] = None, password: Optional[str] = None
@@ -182,7 +192,7 @@ class CSWSearch(Search):
                         e,
                     )
 
-    def __build_product(self, rec: Any, product_type: str, **kwargs: Any) -> EOProduct:
+    def __build_product(self, rec: Any, collection: str, **kwargs: Any) -> EOProduct:
         """Enable search results to be handled by http download plugin"""
         download_url = ""
         resource_filter = re.compile(
@@ -217,11 +227,11 @@ class CSWSearch(Search):
         else:
             properties["geometry"] = wkt.loads(properties["geometry"])
         return EOProduct(
-            product_type,
+            collection,
             self.provider,
             # TODO: EOProduct has no more *args in its __init__ (search_args attribute removed)
             # Not sure why download_url was here in the first place, needs to be updated,
-            # possibly by having instead 'downloadLink' in the properties
+            # possibly by having instead 'eodag:download_link' in the properties
             # download_url,
             properties,
             searched_bbox=kwargs.get("footprints"),
@@ -229,26 +239,26 @@ class CSWSearch(Search):
 
     def __convert_query_params(
         self,
-        product_type_def: dict[str, Any],
-        product_type: str,
+        collection_def: dict[str, Any],
+        collection: str,
         params: dict[str, Any],
     ) -> Union[list[OgcExpression], list[list[OgcExpression]]]:
         """Translates eodag search to CSW constraints using owslib constraint classes"""
         constraints: list[OgcExpression] = []
         # How the match should be performed (fuzzy, prefix, postfix or exact).
         # defaults to fuzzy
-        pt_tag, matching = (
-            product_type_def["name"],
-            product_type_def.get("matching", "fuzzy"),
+        col_tag, matching = (
+            collection_def["name"],
+            collection_def.get("matching", "fuzzy"),
         )
         if matching == "prefix":
-            constraints.append(PropertyIsLike(pt_tag, "{}%".format(product_type)))
+            constraints.append(PropertyIsLike(col_tag, "{}%".format(collection)))
         elif matching == "postfix":
-            constraints.append(PropertyIsLike(pt_tag, "%{}".format(product_type)))
+            constraints.append(PropertyIsLike(col_tag, "%{}".format(collection)))
         elif matching == "exact":
-            constraints.append(PropertyIsEqualTo(pt_tag, product_type))
+            constraints.append(PropertyIsEqualTo(col_tag, collection))
         else:  # unknown matching is considered to be equal to 'fuzzy'
-            constraints.append(PropertyIsLike(pt_tag, "%{}%".format(product_type)))
+            constraints.append(PropertyIsLike(col_tag, "%{}%".format(collection)))
 
         # `footprint`
         fp = params.get("geometry")
@@ -259,8 +269,8 @@ class CSWSearch(Search):
 
         # dates
         start, end = (
-            params.get("startTimeFromAscendingNode"),
-            params.get("completionTimeFromAscendingNode"),
+            params.get("start_datetime"),
+            params.get("end_datetime"),
         )
         if start and "date_tags" in self.config.search_definition:
             constraints.append(