eodag 3.0.0b2__py3-none-any.whl → 3.0.1__py3-none-any.whl

eodag/plugins/search/creodias_s3.py

@@ -29,7 +29,12 @@ from eodag.config import PluginConfig
 from eodag.plugins.authentication.aws_auth import AwsAuth
 from eodag.plugins.search.qssearch import ODataV4Search
 from eodag.utils import guess_file_type
-from eodag.utils.exceptions import AuthenticationError, MisconfiguredError, RequestError
+from eodag.utils.exceptions import (
+    AuthenticationError,
+    MisconfiguredError,
+    NotAvailableError,
+    RequestError,
+)
 
 DATA_EXTENSIONS = ["jp2", "tiff", "nc", "grib"]
 logger = logging.getLogger("eodag.search.creodiass3")
@@ -38,13 +43,10 @@ logger = logging.getLogger("eodag.search.creodiass3")
 def patched_register_downloader(self, downloader, authenticator):
     """Add the download information to the product.
     :param self: product to which information should be added
-    :type self: EoProduct
     :param downloader: The download method that it can use
-    :type downloader: Concrete subclass of
                       :class:`~eodag.plugins.download.base.Download` or
                       :class:`~eodag.plugins.api.base.Api`
     :param authenticator: The authentication method needed to perform the download
-    :type authenticator: Concrete subclass of
                          :class:`~eodag.plugins.authentication.base.Authentication`
     """
     # register downloader
@@ -53,7 +55,7 @@ def patched_register_downloader(self, downloader, authenticator):
     try:
         _update_assets(self, downloader.config, authenticator)
     except BotoCoreError as e:
-        raise RequestError(f"could not update assets: {str(e)}") from e
+        raise RequestError.from_error(e, "could not update assets") from e
 
 
 def _update_assets(product: EOProduct, config: PluginConfig, auth: AwsAuth):
@@ -73,7 +75,7 @@ def _update_assets(product: EOProduct, config: PluginConfig, auth: AwsAuth):
             if not getattr(auth, "s3_client", None):
                 auth.s3_client = boto3.client(
                     "s3",
-                    endpoint_url=config.base_uri,
+                    endpoint_url=config.s3_endpoint,
                     aws_access_key_id=auth_dict["aws_access_key_id"],
                     aws_secret_access_key=auth_dict["aws_secret_access_key"],
                 )
@@ -108,12 +110,22 @@ def _update_assets(product: EOProduct, config: PluginConfig, auth: AwsAuth):
                 raise AuthenticationError(
                     f"Authentication failed on {config.base_uri} s3"
                 ) from e
-            raise RequestError(f"assets for product {prefix} could not be found") from e
+            raise NotAvailableError(
+                f"assets for product {prefix} could not be found"
+            ) from e
 
 
 class CreodiasS3Search(ODataV4Search):
     """
-    Search on creodias and adapt results to s3
+    ``CreodiasS3Search`` is an extension of :class:`~eodag.plugins.search.qssearch.ODataV4Search`,
+    it executes a Search on creodias and adapts results so that the assets contain links to s3.
+    It has the same configuration parameters as :class:`~eodag.plugins.search.qssearch.ODataV4Search` and
+    one additional parameter:
+
+    :param provider: provider name
+    :param config: Search plugin configuration:
+
+        * :attr:`~eodag.config.PluginConfig.s3_endpoint` (``str``) (**mandatory**): base url of the s3
     """
 
     def __init__(self, provider, config):

eodag/plugins/search/csw.py

@@ -52,7 +52,47 @@ SUPPORTED_REFERENCE_SCHEMES = ["WWW:DOWNLOAD-1.0-http--download"]
 
 
 class CSWSearch(Search):
-    """A plugin for implementing search based on OGC CSW"""
+    """A plugin for implementing search based on OGC CSW
+
+    :param provider: provider name
+    :param config: Search plugin configuration:
+
+        * :attr:`~eodag.config.PluginConfig.api_endpoint` (``str``) (**mandatory**): The endpoint of the
+          provider's search interface
+        * :attr:`~eodag.config.PluginConfig.version` (``str``): OGC Catalogue Service version; default: ``2.0.2``
+        * :attr:`~eodag.config.PluginConfig.search_definition` (``Dict[str, Any]``) (**mandatory**):
+
+          * **product_type_tags** (``List[Dict[str, Any]``): dict of product type tags
+          * **resource_location_filter** (``str``): regex string
+          * **date_tags** (``Dict[str, Any]``): tags for start and end
+
+        * :attr:`~eodag.config.PluginConfig.metadata_mapping` (``Dict[str, Any]``): The search plugins of this kind can
+          detect when a metadata mapping is "query-able", and get the semantics of how to format the query string
+          parameter that enables to make a query on the corresponding metadata. To make a metadata query-able,
+          just configure it in the metadata mapping to be a list of 2 items, the first one being the
+          specification of the query string search formatting. The later is a string following the
+          specification of Python string formatting, with a special behaviour added to it. For example,
+          an entry in the metadata mapping of this kind::
+
+                completionTimeFromAscendingNode:
+                    - 'f=acquisition.endViewingDate:lte:{completionTimeFromAscendingNode#timestamp}'
+                    - '$.properties.acquisition.endViewingDate'
+
+          means that the search url will have a query string parameter named ``f`` with a value of
+          ``acquisition.endViewingDate:lte:1543922280.0`` if the search was done with the value
+          of ``completionTimeFromAscendingNode`` being ``2018-12-04T12:18:00``. What happened is that
+          ``{completionTimeFromAscendingNode#timestamp}`` was replaced with the timestamp of the value
+          of ``completionTimeFromAscendingNode``. This example shows all there is to know about the
+          semantics of the query string formatting introduced by this plugin: any eodag search parameter
+          can be referenced in the query string with an additional optional conversion function that
+          is separated from it by a ``#`` (see :func:`~eodag.api.product.metadata_mapping.format_metadata` for further
+          details on the available converters). Note that for the values in the
+          :attr:`~eodag.config.PluginConfig.free_text_search_operations` configuration parameter follow the same rule.
+          If the metadata_mapping is not a list but only a string, this means that the parameters is not queryable but
+          it is included in the result obtained from the provider. The string indicates how the provider result should
+          be mapped to the eodag parameter.
+
+    """
 
     def __init__(self, provider: str, config: PluginConfig) -> None:
         super(CSWSearch, self).__init__(provider, config)

eodag/plugins/search/data_request_search.py

@@ -58,9 +58,107 @@ logger = logging.getLogger("eodag.search.data_request_search")
 class DataRequestSearch(Search):
     """
     Plugin to execute search requests composed of several steps:
-        - do a data request which defines which data shall be searched
-        - check the status of the request job
-        - if finished - fetch the result of the job
+
+    #. do a data request which defines which data shall be searched
+    #. check the status of the request job
+    #. if finished - fetch the result of the job
+
+    :param provider: provider name
+    :param config: Search plugin configuration:
+
+        * :attr:`~eodag.config.PluginConfig.api_endpoint` (``str``) (**mandatory**): The endpoint of the
+          provider's search interface
+        * :attr:`~eodag.config.PluginConfig.results_entry` (``str``) (**mandatory**): The name of
+          the key in the provider search result that gives access to the result entries
+        * :attr:`~eodag.config.PluginConfig.data_request_url` (``str``) (**mandatory**): url
+          to which the data request shall be sent
+        * :attr:`~eodag.config.PluginConfig.status_url` (``str``) (**mandatory**): url to fetch
+          the status of the data request
+        * :attr:`~eodag.config.PluginConfig.result_url` (``str``) (**mandatory**): url to fetch
+          the search result when the data request is done
+        * :attr:`~eodag.config.PluginConfig.need_auth` (``bool``): if authentication is needed for
+          the search request; default: ``False``
+        * :attr:`~eodag.config.PluginConfig.auth_error_code` (``int``): which error code is returned in case of an
+          authentication error; only used if ``need_auth=true``
+        * :attr:`~eodag.config.PluginConfig.ssl_verify` (``bool``): if the ssl certificates should be
+          verified in requests; default: ``True``
+        * :attr:`~eodag.config.PluginConfig.timeout` (``int``): time to wait until request timeout in seconds;
+          default: ``5``
+        * :attr:`~eodag.config.PluginConfig.dates_required` (``bool``): if date parameters are mandatory
+          in the request; default: ``True``
+        * :attr:`~eodag.config.PluginConfig.pagination` (:class:`~eodag.config.PluginConfig.Pagination`)
+          (**mandatory**): The configuration of how the pagination is done on the provider. It is a tree with the
+          following nodes:
+
+          * :attr:`~eodag.config.PluginConfig.Pagination.total_items_nb_key_path` (``str``):  An XPath or JsonPath
+            leading to the total number of results satisfying a request. This is used for providers which provides the
+            total results metadata along with the result of the query and don't have an endpoint for querying
+            the number of items satisfying a request, or for providers for which the count endpoint
+            returns a json or xml document
+          * :attr:`~eodag.config.PluginConfig.Pagination.max_items_per_page` (``int``): The maximum
+            number of items per page that the provider can handle; default: ``50``
+          * :attr:`~eodag.config.PluginConfig.Pagination.start_page` (``int``): number of the
+            first page; default: ``1``
+
+        * :attr:`~eodag.config.PluginConfig.discover_product_types`
+          (:class:`~eodag.config.PluginConfig.DiscoverProductTypes`): configuration for product type discovery based on
+          information from the provider; It contains the keys:
+
+          * :attr:`~eodag.config.PluginConfig.DiscoverProductTypes.fetch_url` (``str``) (**mandatory**): url from which
+            the product types can be fetched
+          * :attr:`~eodag.config.PluginConfig.DiscoverProductTypes.result_type` (``str``): type of the provider result;
+            currently only ``json`` is supported (other types could be used in an extension of this plugin)
+          * :attr:`~eodag.config.PluginConfig.DiscoverProductTypes.results_entry` (``str``) (**mandatory**): json path
+            to the list of product types
+          * :attr:`~eodag.config.PluginConfig.DiscoverProductTypes.generic_product_type_id` (``str``): mapping for the
+            product type id
+          * :attr:`~eodag.config.PluginConfig.DiscoverProductTypes.generic_product_type_parsable_metadata`
+            (``Dict[str, str]``): mapping for product type metadata (e.g. ``abstract``, ``licence``) which can be parsed
+            from the provider result
+          * :attr:`~eodag.config.PluginConfig.DiscoverProductTypes.generic_product_type_parsable_properties`
+            (``Dict[str, str]``): mapping for product type properties which can be parsed from the result that are not
+            product type metadata
+          * :attr:`~eodag.config.PluginConfig.DiscoverProductTypes.single_collection_fetch_url` (``str``): url to fetch
+            data for a single collection; used if product type metadata is not available from the endpoint given in
+            :attr:`~eodag.config.PluginConfig.DiscoverProductTypes.fetch_url`
+          * :attr:`~eodag.config.PluginConfig.DiscoverProductTypes.single_collection_fetch_qs` (``str``): query string
+            to be added to the :attr:`~eodag.config.PluginConfig.DiscoverProductTypes.fetch_url` to filter for a
+            collection
+          * :attr:`~eodag.config.PluginConfig.DiscoverProductTypes.single_product_type_parsable_metadata`
+            (``Dict[str, str]``): mapping for product type metadata returned by the endpoint given in
+            :attr:`~eodag.config.PluginConfig.DiscoverProductTypes.single_collection_fetch_url`.
+
+        * :attr:`~eodag.config.PluginConfig.constraints_file_url` (``str``): url to fetch the constraints for a specific
+          product type, can be an http url or a path to a file; the constraints are used to build queryables
+        * :attr:`~eodag.config.PluginConfig.constraints_entry` (``str``): key in the json result where the constraints
+          can be found; if not given, it is assumed that the constraints are on top level of the result, i.e.
+          the result is an array of constraints
+        * :attr:`~eodag.config.PluginConfig.metadata_mapping` (``Dict[str, Any]``): The search plugins of this kind can
+          detect when a metadata mapping is "query-able", and get the semantics of how to format the query string
+          parameter that enables to make a query on the corresponding metadata. To make a metadata query-able,
+          just configure it in the metadata mapping to be a list of 2 items, the first one being the
+          specification of the query string search formatting. The later is a string following the
+          specification of Python string formatting, with a special behaviour added to it. For example,
+          an entry in the metadata mapping of this kind::
+
+                completionTimeFromAscendingNode:
+                    - 'f=acquisition.endViewingDate:lte:{completionTimeFromAscendingNode#timestamp}'
+                    - '$.properties.acquisition.endViewingDate'
+
+          means that the search url will have a query string parameter named ``f`` with a value of
+          ``acquisition.endViewingDate:lte:1543922280.0`` if the search was done with the value
+          of ``completionTimeFromAscendingNode`` being ``2018-12-04T12:18:00``. What happened is that
+          ``{completionTimeFromAscendingNode#timestamp}`` was replaced with the timestamp of the value
+          of ``completionTimeFromAscendingNode``. This example shows all there is to know about the
+          semantics of the query string formatting introduced by this plugin: any eodag search parameter
+          can be referenced in the query string with an additional optional conversion function that
+          is separated from it by a ``#`` (see :func:`~eodag.api.product.metadata_mapping.format_metadata` for further
+          details on the available converters). Note that for the values in the
+          :attr:`~eodag.config.PluginConfig.free_text_search_operations` configuration parameter follow the same rule.
+          If the metadata_mapping is not a list but only a string, this means that the parameters is not queryable but
+          it is included in the result obtained from the provider. The string indicates how the provider result should
+          be mapped to the eodag parameter.
+
     """
 
     data_request_id: Optional[str]
@@ -116,7 +214,6 @@ class DataRequestSearch(Search):
         """Fetch product types is disabled for `DataRequestSearch`
 
         :returns: empty dict
-        :rtype: (optional) dict
         """
         return None
 
@@ -133,7 +230,7 @@ class DataRequestSearch(Search):
         """
         performs the search for a provider where several steps are required to fetch the data
         """
-        if kwargs.get("sortBy"):
+        if kwargs.get("sort_by"):
             raise ValidationError(f"{self.provider} does not support sorting feature")
 
         product_type = kwargs.get("productType", None)
@@ -276,9 +373,9 @@ class DataRequestSearch(Search):
         except requests.exceptions.Timeout as exc:
             raise TimeOutError(exc, timeout=HTTP_REQ_TIMEOUT) from exc
         except requests.RequestException as e:
-            raise RequestError(
-                f"search job for product_type {product_type} could not be created: {str(e)}, {request_job.text}"
-            )
+            raise RequestError.from_error(
+                e, f"search job for product_type {product_type} could not be created"
+            ) from e
         else:
             logger.info("search job for product_type %s created", product_type)
             return request_job.json()["jobId"]
@@ -295,7 +392,7 @@ class DataRequestSearch(Search):
         except requests.exceptions.Timeout as exc:
             raise TimeOutError(exc, timeout=HTTP_REQ_TIMEOUT) from exc
         except requests.RequestException as e:
-            raise RequestError(f"_cancel_request failed: {str(e)}")
+            raise RequestError.from_error(e, "_cancel_request failed") from e
 
     def _check_request_status(self, data_request_id: str) -> bool:
         logger.debug("checking status of request job %s", data_request_id)
@@ -314,7 +411,7 @@ class DataRequestSearch(Search):
         except requests.exceptions.Timeout as exc:
             raise TimeOutError(exc, timeout=HTTP_REQ_TIMEOUT) from exc
         except requests.RequestException as e:
-            raise RequestError(f"_check_request_status failed: {str(e)}")
+            raise RequestError.from_error(e, "_check_request_status failed") from e
         else:
             status_data = status_resp.json()
             if "status_code" in status_data and status_data["status_code"] in [
@@ -322,7 +419,9 @@ class DataRequestSearch(Search):
                 404,
             ]:
                 logger.error(f"_check_request_status failed: {status_data}")
-                raise RequestError("authentication token expired during request")
+                error = RequestError("authentication token expired during request")
+                error.status_code = status_data["status_code"]
+                raise error
             if status_data["status"] == "failed":
                 logger.error(f"_check_request_status failed: {status_data}")
                 raise RequestError(
@@ -386,8 +485,11 @@ class DataRequestSearch(Search):
         total_items_nb_key_path = string_to_jsonpath(
             self.config.pagination["total_items_nb_key_path"]
         )
-        if len(total_items_nb_key_path.find(results)) > 0:
-            total_items_nb = total_items_nb_key_path.find(results)[0].value
+        found_total_items_nb_paths = total_items_nb_key_path.find(results)
+        if found_total_items_nb_paths and not isinstance(
+            found_total_items_nb_paths, int
+        ):
+            total_items_nb = found_total_items_nb_paths[0].value
         else:
             total_items_nb = 0
         for p in products:
@@ -423,7 +525,10 @@ class DataRequestSearch(Search):
         path = string_to_jsonpath(custom_filters["filter_attribute"])
         indexes = custom_filters["indexes"].split("-")
         for record in results:
-            filter_param = path.find(record)[0].value
+            found_paths = path.find(record)
+            if not found_paths or isinstance(found_paths, int):
+                continue
+            filter_param = found_paths[0].value
             filter_value = filter_param[int(indexes[0]) : int(indexes[1])]
             filter_clause = "'" + filter_value + "' " + custom_filters["filter_clause"]
             if eval(filter_clause):