eodag 3.0.0b3__py3-none-any.whl → 3.1.0__py3-none-any.whl

eodag/plugins/search/data_request_search.py

@@ -20,7 +20,7 @@ from __future__ import annotations
 import logging
 import time
 from datetime import datetime, timedelta, timezone
-from typing import TYPE_CHECKING, Any, Dict, List, Optional, Tuple, cast
+from typing import TYPE_CHECKING, Any, Optional, cast
 
 import requests
 
@@ -36,6 +36,7 @@ from eodag.utils import (
     DEFAULT_ITEMS_PER_PAGE,
     DEFAULT_MISSION_START_DATE,
     DEFAULT_PAGE,
+    DEFAULT_SEARCH_TIMEOUT,
     GENERIC_PRODUCT_TYPE,
     HTTP_REQ_TIMEOUT,
     USER_AGENT,
@@ -58,9 +59,107 @@ logger = logging.getLogger("eodag.search.data_request_search")
 class DataRequestSearch(Search):
     """
     Plugin to execute search requests composed of several steps:
-        - do a data request which defines which data shall be searched
-        - check the status of the request job
-        - if finished - fetch the result of the job
+
+    #. do a data request which defines which data shall be searched
+    #. check the status of the request job
+    #. if finished - fetch the result of the job
+
+    :param provider: provider name
+    :param config: Search plugin configuration:
+
+        * :attr:`~eodag.config.PluginConfig.api_endpoint` (``str``) (**mandatory**): The endpoint of the
+          provider's search interface
+        * :attr:`~eodag.config.PluginConfig.results_entry` (``str``) (**mandatory**): The name of
+          the key in the provider search result that gives access to the result entries
+        * :attr:`~eodag.config.PluginConfig.data_request_url` (``str``) (**mandatory**): url
+          to which the data request shall be sent
+        * :attr:`~eodag.config.PluginConfig.status_url` (``str``) (**mandatory**): url to fetch
+          the status of the data request
+        * :attr:`~eodag.config.PluginConfig.result_url` (``str``) (**mandatory**): url to fetch
+          the search result when the data request is done
+        * :attr:`~eodag.config.PluginConfig.need_auth` (``bool``): if authentication is needed for
+          the search request; default: ``False``
+        * :attr:`~eodag.config.PluginConfig.auth_error_code` (``int``): which error code is returned in case of an
+          authentication error; only used if ``need_auth=true``
+        * :attr:`~eodag.config.PluginConfig.ssl_verify` (``bool``): if the ssl certificates should be
+          verified in requests; default: ``True``
+        * :attr:`~eodag.config.PluginConfig.timeout` (``int``): time to wait until request timeout in seconds;
+          default: ``5``
+        * :attr:`~eodag.config.PluginConfig.dates_required` (``bool``): if date parameters are mandatory
+          in the request; default: ``True``
+        * :attr:`~eodag.config.PluginConfig.pagination` (:class:`~eodag.config.PluginConfig.Pagination`)
+          (**mandatory**): The configuration of how the pagination is done on the provider. It is a tree with the
+          following nodes:
+
+          * :attr:`~eodag.config.PluginConfig.Pagination.total_items_nb_key_path` (``str``):  An XPath or JsonPath
+            leading to the total number of results satisfying a request. This is used for providers which provides the
+            total results metadata along with the result of the query and don't have an endpoint for querying
+            the number of items satisfying a request, or for providers for which the count endpoint
+            returns a json or xml document
+          * :attr:`~eodag.config.PluginConfig.Pagination.max_items_per_page` (``int``): The maximum
+            number of items per page that the provider can handle; default: ``50``
+          * :attr:`~eodag.config.PluginConfig.Pagination.start_page` (``int``): number of the
+            first page; default: ``1``
+
+        * :attr:`~eodag.config.PluginConfig.discover_product_types`
+          (:class:`~eodag.config.PluginConfig.DiscoverProductTypes`): configuration for product type discovery based on
+          information from the provider; It contains the keys:
+
+          * :attr:`~eodag.config.PluginConfig.DiscoverProductTypes.fetch_url` (``str``) (**mandatory**): url from which
+            the product types can be fetched
+          * :attr:`~eodag.config.PluginConfig.DiscoverProductTypes.result_type` (``str``): type of the provider result;
+            currently only ``json`` is supported (other types could be used in an extension of this plugin)
+          * :attr:`~eodag.config.PluginConfig.DiscoverProductTypes.results_entry` (``str``) (**mandatory**): json path
+            to the list of product types
+          * :attr:`~eodag.config.PluginConfig.DiscoverProductTypes.generic_product_type_id` (``str``): mapping for the
+            product type id
+          * :attr:`~eodag.config.PluginConfig.DiscoverProductTypes.generic_product_type_parsable_metadata`
+            (``dict[str, str]``): mapping for product type metadata (e.g. ``abstract``, ``licence``) which can be parsed
+            from the provider result
+          * :attr:`~eodag.config.PluginConfig.DiscoverProductTypes.generic_product_type_parsable_properties`
+            (``dict[str, str]``): mapping for product type properties which can be parsed from the result and are not
+            product type metadata
+          * :attr:`~eodag.config.PluginConfig.DiscoverProductTypes.single_collection_fetch_url` (``str``): url to fetch
+            data for a single collection; used if product type metadata is not available from the endpoint given in
+            :attr:`~eodag.config.PluginConfig.DiscoverProductTypes.fetch_url`
+          * :attr:`~eodag.config.PluginConfig.DiscoverProductTypes.single_collection_fetch_qs` (``str``): query string
+            to be added to the :attr:`~eodag.config.PluginConfig.DiscoverProductTypes.fetch_url` to filter for a
+            collection
+          * :attr:`~eodag.config.PluginConfig.DiscoverProductTypes.single_product_type_parsable_metadata`
+            (``dict[str, str]``): mapping for product type metadata returned by the endpoint given in
+            :attr:`~eodag.config.PluginConfig.DiscoverProductTypes.single_collection_fetch_url`.
+
+        * :attr:`~eodag.config.PluginConfig.constraints_file_url` (``str``): url to fetch the constraints for a specific
+          product type, can be an http url or a path to a file; the constraints are used to build queryables
+        * :attr:`~eodag.config.PluginConfig.constraints_entry` (``str``): key in the json result where the constraints
+          can be found; if not given, it is assumed that the constraints are on top level of the result, i.e.
+          the result is an array of constraints
+        * :attr:`~eodag.config.PluginConfig.metadata_mapping` (``dict[str, Any]``): The search plugins of this kind can
+          detect when a metadata mapping is "query-able", and get the semantics of how to format the query string
+          parameter that enables to make a query on the corresponding metadata. To make a metadata query-able,
+          just configure it in the metadata mapping to be a list of 2 items, the first one being the
+          specification of the query string search formatting. The later is a string following the
+          specification of Python string formatting, with a special behaviour added to it. For example,
+          an entry in the metadata mapping of this kind::
+
+                completionTimeFromAscendingNode:
+                    - 'f=acquisition.endViewingDate:lte:{completionTimeFromAscendingNode#timestamp}'
+                    - '$.properties.acquisition.endViewingDate'
+
+          means that the search url will have a query string parameter named ``f`` with a value of
+          ``acquisition.endViewingDate:lte:1543922280.0`` if the search was done with the value
+          of ``completionTimeFromAscendingNode`` being ``2018-12-04T12:18:00``. What happened is that
+          ``{completionTimeFromAscendingNode#timestamp}`` was replaced with the timestamp of the value
+          of ``completionTimeFromAscendingNode``. This example shows all there is to know about the
+          semantics of the query string formatting introduced by this plugin: any eodag search parameter
+          can be referenced in the query string with an additional optional conversion function that
+          is separated from it by a ``#`` (see :func:`~eodag.api.product.metadata_mapping.format_metadata` for further
+          details on the available converters). Note that for the values in the
+          :attr:`~eodag.config.PluginConfig.free_text_search_operations` configuration parameter follow the same rule.
+          If the metadata_mapping is not a list but only a string, this means that the parameters is not queryable but
+          it is included in the result obtained from the provider. The string indicates how the provider result should
+          be mapped to the eodag parameter.
+
     """
 
     data_request_id: Optional[str]
@@ -88,7 +187,7 @@ class DataRequestSearch(Search):
                 )
                 if other_product_for_mapping:
                     other_product_type_def_params = self.get_product_type_def_params(
-                        other_product_for_mapping,  # **kwargs
+                        other_product_for_mapping,
                     )
                     product_type_metadata_mapping.update(
                         other_product_type_def_params.get("metadata_mapping", {})
@@ -109,10 +208,10 @@ class DataRequestSearch(Search):
             self.config.pagination["next_page_url_key_path"] = string_to_jsonpath(
                 self.config.pagination.get("next_page_url_key_path", None)
             )
-        self.download_info: Dict[str, Any] = {}
+        self.download_info: dict[str, Any] = {}
         self.data_request_id = None
 
-    def discover_product_types(self, **kwargs: Any) -> Optional[Dict[str, Any]]:
+    def discover_product_types(self, **kwargs: Any) -> Optional[dict[str, Any]]:
         """Fetch product types is disabled for `DataRequestSearch`
 
         :returns: empty dict
@@ -128,7 +227,7 @@ class DataRequestSearch(Search):
         self,
         prep: PreparedSearch = PreparedSearch(),
         **kwargs: Any,
-    ) -> Tuple[List[EOProduct], Optional[int]]:
+    ) -> tuple[list[EOProduct], Optional[int]]:
         """
         performs the search for a provider where several steps are required to fetch the data
         """
@@ -154,7 +253,7 @@ class DataRequestSearch(Search):
 
         # provider product type specific conf
         self.product_type_def_params = self.get_product_type_def_params(
-            product_type, **kwargs
+            product_type, format_variables=kwargs
         )
 
         # update config using provider product type definition metadata_mapping
@@ -164,7 +263,7 @@ class DataRequestSearch(Search):
         )
         if other_product_for_mapping:
             other_product_type_def_params = self.get_product_type_def_params(
-                other_product_for_mapping, **kwargs
+                other_product_for_mapping, format_variables=kwargs
             )
             self.config.metadata_mapping.update(
                 other_product_type_def_params.get("metadata_mapping", {})
@@ -210,7 +309,7 @@ class DataRequestSearch(Search):
             request_finished = True
 
         # loop to check search job status
-        search_timeout = int(getattr(self.config, "timeout", HTTP_REQ_TIMEOUT))
+        search_timeout = int(getattr(self.config, "timeout", DEFAULT_SEARCH_TIMEOUT))
         logger.info(
             f"checking status of request job {data_request_id} (timeout={search_timeout}s)"
         )
@@ -275,9 +374,9 @@ class DataRequestSearch(Search):
         except requests.exceptions.Timeout as exc:
             raise TimeOutError(exc, timeout=HTTP_REQ_TIMEOUT) from exc
         except requests.RequestException as e:
-            raise RequestError(
-                f"search job for product_type {product_type} could not be created: {str(e)}, {request_job.text}"
-            )
+            raise RequestError.from_error(
+                e, f"search job for product_type {product_type} could not be created"
+            ) from e
         else:
             logger.info("search job for product_type %s created", product_type)
             return request_job.json()["jobId"]
@@ -294,7 +393,7 @@ class DataRequestSearch(Search):
         except requests.exceptions.Timeout as exc:
             raise TimeOutError(exc, timeout=HTTP_REQ_TIMEOUT) from exc
         except requests.RequestException as e:
-            raise RequestError(f"_cancel_request failed: {str(e)}")
+            raise RequestError.from_error(e, "_cancel_request failed") from e
 
     def _check_request_status(self, data_request_id: str) -> bool:
         logger.debug("checking status of request job %s", data_request_id)
@@ -313,7 +412,7 @@ class DataRequestSearch(Search):
         except requests.exceptions.Timeout as exc:
             raise TimeOutError(exc, timeout=HTTP_REQ_TIMEOUT) from exc
         except requests.RequestException as e:
-            raise RequestError(f"_check_request_status failed: {str(e)}")
+            raise RequestError.from_error(e, "_check_request_status failed") from e
         else:
             status_data = status_resp.json()
             if "status_code" in status_data and status_data["status_code"] in [
@@ -321,7 +420,9 @@ class DataRequestSearch(Search):
                 404,
             ]:
                 logger.error(f"_check_request_status failed: {status_data}")
-                raise RequestError("authentication token expired during request")
+                error = RequestError("authentication token expired during request")
+                error.status_code = status_data["status_code"]
+                raise error
             if status_data["status"] == "failed":
                 logger.error(f"_check_request_status failed: {status_data}")
                 raise RequestError(
@@ -331,7 +432,7 @@ class DataRequestSearch(Search):
 
     def _get_result_data(
         self, data_request_id: str, items_per_page: int, page: int
-    ) -> Dict[str, Any]:
+    ) -> dict[str, Any]:
         page = page - 1 + self.config.pagination.get("start_page", 1)
         url = self.config.result_url.format(
             jobId=data_request_id, items_per_page=items_per_page, page=page
@@ -350,18 +451,18 @@ class DataRequestSearch(Search):
 
     def _convert_result_data(
         self,
-        result_data: Dict[str, Any],
+        result_data: dict[str, Any],
         data_request_id: str,
         product_type: str,
         **kwargs: Any,
-    ) -> Tuple[List[EOProduct], int]:
+    ) -> tuple[list[EOProduct], int]:
         """Build EOProducts from provider results"""
         results_entry = self.config.results_entry
         results = result_data[results_entry]
         logger.debug(
             "Adapting %s plugin results to eodag product representation" % len(results)
         )
-        products: List[EOProduct] = []
+        products: list[EOProduct] = []
         for result in results:
             product = EOProduct(
                 self.provider,
@@ -417,8 +518,8 @@ class DataRequestSearch(Search):
         return False
 
     def _apply_additional_filters(
-        self, result: Dict[str, Any], custom_filters: Dict[str, str]
-    ) -> Dict[str, Any]:
+        self, result: dict[str, Any], custom_filters: dict[str, str]
+    ) -> dict[str, Any]:
         filtered_result = []
         results_entry = self.config.results_entry
         results = result[results_entry]