eodag 3.0.0b3__py3-none-any.whl → 3.0.1__py3-none-any.whl

eodag/plugins/search/qssearch.py

@@ -20,8 +20,10 @@ from __future__ import annotations
 import logging
 import re
 from copy import copy as copy_copy
+from datetime import datetime
 from typing import (
     TYPE_CHECKING,
+    Annotated,
     Any,
     Callable,
     Dict,
@@ -32,6 +34,7 @@ from typing import (
     Tuple,
     TypedDict,
     cast,
+    get_args,
 )
 from urllib.error import URLError
 from urllib.parse import (
@@ -44,10 +47,12 @@ from urllib.parse import (
 )
 from urllib.request import Request, urlopen
 
+import concurrent.futures
 import geojson
 import orjson
 import requests
 import yaml
+from dateutil.utils import today
 from jsonpath_ng import JSONPath
 from lxml import etree
 from pydantic import create_model
@@ -55,6 +60,7 @@ from pydantic.fields import FieldInfo
 from requests import Response
 from requests.adapters import HTTPAdapter
 from requests.auth import AuthBase
+from urllib3 import Retry
 
 from eodag.api.product import EOProduct
 from eodag.api.product.metadata_mapping import (
@@ -74,13 +80,14 @@ from eodag.types.search_args import SortByList
 from eodag.utils import (
     GENERIC_PRODUCT_TYPE,
     HTTP_REQ_TIMEOUT,
+    REQ_RETRY_BACKOFF_FACTOR,
+    REQ_RETRY_STATUS_FORCELIST,
+    REQ_RETRY_TOTAL,
     USER_AGENT,
-    Annotated,
     _deprecated,
     deepcopy,
     dict_items_recursive_apply,
     format_dict_items,
-    get_args,
     get_ssl_context,
     quote,
     string_to_jsonpath,
@@ -108,98 +115,180 @@ logger = logging.getLogger("eodag.search.qssearch")
 
 class QueryStringSearch(Search):
     """A plugin that helps implementing any kind of search protocol that relies on
-    query strings (e.g: opensearch).
-
-    The available configuration parameters for this kind of plugin are:
-
-        - **result_type**: (optional) One of "json" or "xml", depending on the
-          representation of the provider's search results. The default is "json"
-
-        - **results_entry**: (mandatory) The name of the key in the provider search
-          result that gives access to the result entries
-
-        - **api_endpoint**: (mandatory) The endpoint of the provider's search interface
-
-        - **literal_search_params**: (optional) A mapping of (search_param =>
-          search_value) pairs giving search parameters to be passed as is in the search
-          url query string. This is useful for example in situations where the user wants
-          to pass-in a search query as it is done on the provider interface. In such a case,
-          the user can put in his configuration file the query he needs to pass to the provider.
-
-        - **pagination**: (mandatory) The configuration of how the pagination is done
-          on the provider. It is a tree with the following nodes:
-
-          - *next_page_url_tpl*: The template for pagination requests. This is a simple
-            Python format string which will be resolved using the following keywords:
-            ``url`` (the base url of the search endpoint), ``search`` (the query string
-            corresponding to the search request), ``items_per_page`` (the number of
-            items to return per page), ``skip`` (the number of items to skip) or
-            ``skip_base_1`` (the number of items to skip, starting from 1) and
-            ``page`` (which page to return).
-
-          - *total_items_nb_key_path*: (optional) An XPath or JsonPath leading to the
-            total number of results satisfying a request. This is used for providers
-            which provides the total results metadata along with the result of the
-            query and don't have an endpoint for querying the number of items
-            satisfying a request, or for providers for which the count endpoint returns
-            a json or xml document
-
-          - *count_endpoint*: (optional) The endpoint for counting the number of items
-            satisfying a request
-
-          - *next_page_url_key_path*: (optional) A JSONPATH expression used to retrieve
-            the URL of the next page in the response of the current page.
-
-        - **free_text_search_operations**: (optional) A tree structure of the form::
-
-            # noqa: E800
-            <search-param>:     # e.g: $search
-                union: # how to join the operations below (e.g: ' AND ' -->
-                       # '(op1 AND op2) AND (op3 OR op4)')
-                wrapper: # a pattern for how each operation will be wrapped
-                         # (e.g: '({})' --> '(op1 AND op2)')
-                operations:     # The operations to build
-                  <opname>:     # e.g: AND
-                    - <op1>    # e.g:
-                               # 'sensingStartDate:[{startTimeFromAscendingNode}Z TO *]'
-                    - <op2>    # e.g:
-                        # 'sensingStopDate:[* TO {completionTimeFromAscendingNode}Z]'
-                    ...
-                  ...
-            ...
-
-          With the structure above, each operation will become a string of the form:
-          '(<op1> <opname> <op2>)', then the operations will be joined together using
-          the union string and finally if the number of operations is greater than 1,
-          they will be wrapped as specified by the wrapper config key.
-
-    The search plugins of this kind can detect when a metadata mapping is "query-able",
-    and get the semantics of how to format the query string parameter that enables to
-    make a query on the corresponding metadata. To make a metadata query-able, just
-    configure it in the metadata mapping to be a list of 2 items, the first one being
-    the specification of the query string search formatting. The later is a string
-    following the specification of Python string formatting, with a special behaviour
-    added to it. For example, an entry in the metadata mapping of this kind::
-
-        completionTimeFromAscendingNode:
-            - 'f=acquisition.endViewingDate:lte:{completionTimeFromAscendingNode#timestamp}'
-            - '$.properties.acquisition.endViewingDate'
-
-    means that the search url will have a query string parameter named *"f"* with a
-    value of *"acquisition.endViewingDate:lte:1543922280.0"* if the search was done
-    with the value of ``completionTimeFromAscendingNode`` being
-    ``2018-12-04T12:18:00``. What happened is that
-    ``{completionTimeFromAscendingNode#timestamp}`` was replaced with the timestamp
-    of the value of ``completionTimeFromAscendingNode``. This example shows all there
-    is to know about the semantics of the query string formatting introduced by this
-    plugin: any eodag search parameter can be referenced in the query string
-    with an additional optional conversion function that is separated from it by a
-    ``#`` (see :func:`~eodag.utils.format_metadata` for further details on the
-    available converters). Note that for the values in the
-    ``free_text_search_operations`` configuration parameter follow the same rule.
-
-    :param provider: An eodag providers configuration dictionary
-    :param config: Path to the user configuration file
+    query strings (e.g: opensearch). Most of the other search plugins inherit from this plugin.
+
+    :param provider: provider name
+    :param config: Search plugin configuration:
+
+        * :attr:`~eodag.config.PluginConfig.result_type` (``str``): One of ``json`` or ``xml``, depending on the
+          representation of the provider's search results. The default is ``json``.
+        * :attr:`~eodag.config.PluginConfig.results_entry` (``str``) (**mandatory**): The name of the key in the
+          provider search result that gives access to the result entries
+        * :attr:`~eodag.config.PluginConfig.api_endpoint` (``str``) (**mandatory**): The endpoint of the provider's
+          search interface
+        * :attr:`~eodag.config.PluginConfig.need_auth` (``bool``): if authentication is needed for the search request;
+          default: ``False``
+        * :attr:`~eodag.config.PluginConfig.auth_error_code` (``int``): which error code is returned in case of an
+          authentication error; only used if ``need_auth=true``
+        * :attr:`~eodag.config.PluginConfig.ssl_verify` (``bool``): if the ssl certificates should be verified in
+          requests; default: ``True``
+        * :attr:`~eodag.config.PluginConfig.dont_quote` (``List[str]``): characters that should not be quoted in the
+          url params
+        * :attr:`~eodag.config.PluginConfig.timeout` (``int``): time to wait until request timeout in seconds;
+          default: ``5``
+        * :attr:`~eodag.config.PluginConfig.retry_total` (``int``): :class:`urllib3.util.Retry` ``total`` parameter,
+          total number of retries to allow; default: ``3``
+        * :attr:`~eodag.config.PluginConfig.retry_backoff_factor` (``int``): :class:`urllib3.util.Retry`
+          ``backoff_factor`` parameter, backoff factor to apply between attempts after the second try; default: ``2``
+        * :attr:`~eodag.config.PluginConfig.retry_status_forcelist` (``List[int]``): :class:`urllib3.util.Retry`
+          ``status_forcelist`` parameter, list of integer HTTP status codes that we should force a retry on; default:
+          ``[401, 429, 500, 502, 503, 504]``
+        * :attr:`~eodag.config.PluginConfig.literal_search_params` (``Dict[str, str]``): A mapping of (search_param =>
+          search_value) pairs giving search parameters to be passed as is in the search url query string. This is useful
+          for example in situations where the user wants to add a fixed search query parameter exactly
+          as it is done on the provider interface.
+        * :attr:`~eodag.config.PluginConfig.pagination` (:class:`~eodag.config.PluginConfig.Pagination`)
+          (**mandatory**): The configuration of how the pagination is done on the provider. It is a tree with the
+          following nodes:
+
+          * :attr:`~eodag.config.PluginConfig.Pagination.next_page_url_tpl` (``str``) (**mandatory**): The template for
+            pagination requests. This is a simple Python format string which will be resolved using the following
+            keywords: ``url`` (the base url of the search endpoint), ``search`` (the query string corresponding
+            to the search request), ``items_per_page`` (the number of items to return per page),
+            ``skip`` (the number of items to skip) or ``skip_base_1`` (the number of items to skip,
+            starting from 1) and ``page`` (which page to return).
+          * :attr:`~eodag.config.PluginConfig.Pagination.total_items_nb_key_path` (``str``):  An XPath or JsonPath
+            leading to the total number of results satisfying a request. This is used for providers which provides the
+            total results metadata along with the result of the query and don't have an endpoint for querying
+            the number of items satisfying a request, or for providers for which the count endpoint
+            returns a json or xml document
+          * :attr:`~eodag.config.PluginConfig.Pagination.count_endpoint` (``str``): The endpoint for counting the number
+            of items satisfying a request
+          * :attr:`~eodag.config.PluginConfig.Pagination.count_tpl` (``str``): template for the count parameter that
+            should be added to the search request
+          * :attr:`~eodag.config.PluginConfig.Pagination.next_page_url_key_path` (``str``): A JsonPath expression used
+            to retrieve the URL of the next page in the response of the current page.
+          * :attr:`~eodag.config.PluginConfig.Pagination.max_items_per_page` (``int``): The maximum number of items per
+            page that the provider can handle; default: ``50``
+          * :attr:`~eodag.config.PluginConfig.Pagination.start_page` (``int``): number of the first page; default: ``1``
+
+        * :attr:`~eodag.config.PluginConfig.discover_product_types`
+          (:class:`~eodag.config.PluginConfig.DiscoverProductTypes`): configuration for product type discovery based on
+          information from the provider; It contains the keys:
+
+          * :attr:`~eodag.config.PluginConfig.DiscoverProductTypes.fetch_url` (``str``) (**mandatory**): url from which
+            the product types can be fetched
+          * :attr:`~eodag.config.PluginConfig.DiscoverProductTypes.max_connections` (``int``): Maximum number of
+            connections for concurrent HTTP requests
+          * :attr:`~eodag.config.PluginConfig.DiscoverProductTypes.result_type` (``str``): type of the provider result;
+            currently only ``json`` is supported (other types could be used in an extension of this plugin)
+          * :attr:`~eodag.config.PluginConfig.DiscoverProductTypes.results_entry` (``str``) (**mandatory**): json path
+            to the list of product types
+          * :attr:`~eodag.config.PluginConfig.DiscoverProductTypes.generic_product_type_id` (``str``): mapping for the
+            product type id
+          * :attr:`~eodag.config.PluginConfig.DiscoverProductTypes.generic_product_type_parsable_metadata`
+            (``Dict[str, str]``): mapping for product type metadata (e.g. ``abstract``, ``licence``) which can be parsed
+            from the provider result
+          * :attr:`~eodag.config.PluginConfig.DiscoverProductTypes.generic_product_type_parsable_properties`
+            (``Dict[str, str]``): mapping for product type properties which can be parsed from the result that are not
+            product type metadata
+          * :attr:`~eodag.config.PluginConfig.DiscoverProductTypes.single_collection_fetch_url` (``str``): url to fetch
+            data for a single collection; used if product type metadata is not available from the endpoint given in
+            :attr:`~eodag.config.PluginConfig.DiscoverProductTypes.fetch_url`
+          * :attr:`~eodag.config.PluginConfig.DiscoverProductTypes.single_collection_fetch_qs` (``str``): query string
+            to be added to the :attr:`~eodag.config.PluginConfig.DiscoverProductTypes.fetch_url` to filter for a
+            collection
+          * :attr:`~eodag.config.PluginConfig.DiscoverProductTypes.single_product_type_parsable_metadata`
+            (``Dict[str, str]``): mapping for product type metadata returned by the endpoint given in
+            :attr:`~eodag.config.PluginConfig.DiscoverProductTypes.single_collection_fetch_url`.
+
+        * :attr:`~eodag.config.PluginConfig.sort` (:class:`~eodag.config.PluginConfig.Sort`): configuration for sorting
+          the results. It contains the keys:
+
+          * :attr:`~eodag.config.PluginConfig.Sort.sort_by_default` (``List[Tuple(str, Literal["ASC", "DESC"])]``):
+            parameter and sort order by which the result will be sorted by default (if the user does not enter a
+            ``sort_by`` parameter); if not given the result will use the default sorting of the provider; Attention:
+            for some providers sorting might cause a timeout if no filters are used. In that case no default
+            sort parameters should be given. The format is::
+
+                  sort_by_default:
+                      - !!python/tuple [<param>, <sort order> (ASC or DESC)]
+
+          * :attr:`~eodag.config.PluginConfig.Sort.sort_by_tpl` (``str``): template for the sort parameter that is added
+            to the request; It contains the parameters `sort_param` and `sort_order` which will be replaced by user
+            input or default value. If the parameters are added as query params to a GET request, the string
+            should start with ``&``, otherwise it should be a valid json string surrounded by ``{{ }}``.
+          * :attr:`~eodag.config.PluginConfig.Sort.sort_param_mapping` (``Dict [str, str]``): mapping for the parameters
+            available for sorting
+          * :attr:`~eodag.config.PluginConfig.Sort.sort_order_mapping`
+            (``Dict[Literal["ascending", "descending"], str]``): mapping for the sort order
+          * :attr:`~eodag.config.PluginConfig.Sort.max_sort_params` (``int``): maximum number of sort parameters
+            supported by the provider; used to validate the user input to avoid failed requests or unexpected behaviour
+            (not all parameters are used in the request)
+
+        * :attr:`~eodag.config.PluginConfig.metadata_mapping` (``Dict[str, Any]``): The search plugins of this kind can
+          detect when a metadata mapping is "query-able", and get the semantics of how to format the query string
+          parameter that enables to make a query on the corresponding metadata. To make a metadata query-able,
+          just configure it in the metadata mapping to be a list of 2 items, the first one being the
+          specification of the query string search formatting. The later is a string following the
+          specification of Python string formatting, with a special behaviour added to it. For example,
+          an entry in the metadata mapping of this kind::
+
+                completionTimeFromAscendingNode:
+                    - 'f=acquisition.endViewingDate:lte:{completionTimeFromAscendingNode#timestamp}'
+                    - '$.properties.acquisition.endViewingDate'
+
+          means that the search url will have a query string parameter named ``f`` with a value of
+          ``acquisition.endViewingDate:lte:1543922280.0`` if the search was done with the value
+          of ``completionTimeFromAscendingNode`` being ``2018-12-04T12:18:00``. What happened is that
+          ``{completionTimeFromAscendingNode#timestamp}`` was replaced with the timestamp of the value
+          of ``completionTimeFromAscendingNode``. This example shows all there is to know about the
+          semantics of the query string formatting introduced by this plugin: any eodag search parameter
+          can be referenced in the query string with an additional optional conversion function that
+          is separated from it by a ``#`` (see :func:`~eodag.api.product.metadata_mapping.format_metadata` for further
+          details on the available converters). Note that for the values in the
+          :attr:`~eodag.config.PluginConfig.free_text_search_operations` configuration parameter follow the same rule.
+          If the metadata_mapping is not a list but only a string, this means that the parameters is not queryable but
+          it is included in the result obtained from the provider. The string indicates how the provider result should
+          be mapped to the eodag parameter.
+        * :attr:`~eodag.config.PluginConfig.discover_metadata` (:class:`~eodag.config.PluginConfig.DiscoverMetadata`):
+          configuration for the auto-discovery of queryable parameters as well as parameters returned by the provider
+          which are not in the metadata mapping. It has the attributes:
+
+          * :attr:`~eodag.config.PluginConfig.DiscoverMetadata.auto_discovery` (``bool``): if the automatic discovery of
+            metadata is activated; default: ``False``; if false, the other parameters are not used;
+          * :attr:`~eodag.config.PluginConfig.DiscoverMetadata.metadata_pattern` (``str``): regex string a parameter in
+            the result should match so that is used
+          * :attr:`~eodag.config.PluginConfig.DiscoverMetadata.search_param` (``Union [str, Dict[str, Any]]``): format
+            to add a query param given by the user and not in the metadata mapping to the requests, 'metadata' will be
+            replaced by the search param; can be a string or a dict containing
+            :attr:`~eodag.config.PluginConfig.free_text_search_operations`
+            (see :class:`~eodag.plugins.search.qssearch.ODataV4Search`)
+          * :attr:`~eodag.config.PluginConfig.DiscoverMetadata.metadata_path` (``str``): path where the queryable
+            properties can be found in the provider result
+
+        * :attr:`~eodag.config.PluginConfig.discover_queryables`
+          (:class:`~eodag.config.PluginConfig.DiscoverQueryables`): configuration to fetch the queryables from a
+          provider queryables endpoint; It has the following keys:
+
+          * :attr:`~eodag.config.PluginConfig.DiscoverQueryables.fetch_url` (``str``): url to fetch the queryables valid
+            for all product types
+          * :attr:`~eodag.config.PluginConfig.DiscoverQueryables.product_type_fetch_url` (``str``): url to fetch the
+            queryables for a specific product type
+          * :attr:`~eodag.config.PluginConfig.DiscoverQueryables.result_type` (``str``): type of the result (currently
+            only ``json`` is used)
+          * :attr:`~eodag.config.PluginConfig.DiscoverQueryables.results_entry` (``str``): json path to retrieve the
+            queryables from the provider result
+
+        * :attr:`~eodag.config.PluginConfig.constraints_file_url` (``str``): url to fetch the constraints for a specific
+          product type, can be an http url or a path to a file; the constraints are used to build queryables
+        * :attr:`~eodag.config.PluginConfig.constraints_file_dataset_key` (``str``): key which is used in the eodag
+          configuration to map the eodag product type to the provider product type; default: ``dataset``
+        * :attr:`~eodag.config.PluginConfig.constraints_entry` (``str``): key in the json result where the constraints
+          can be found; if not given, it is assumed that the constraints are on top level of the result, i.e.
+          the result is an array of constraints
+        * :attr:`~eodag.config.PluginConfig.stop_without_constraints_entry_key` (``bool``): if true only a provider
+          result containing `constraints_entry` is accepted as valid and used to create constraints; default: ``False``
     """
 
     extract_properties: Dict[str, Callable[..., Dict[str, Any]]] = {
@@ -362,17 +451,70 @@ class QueryStringSearch(Search):
     def discover_product_types(self, **kwargs: Any) -> Optional[Dict[str, Any]]:
         """Fetch product types list from provider using `discover_product_types` conf
 
+        :returns: configuration dict containing fetched product types information
+        """
+        unpaginated_fetch_url = self.config.discover_product_types.get("fetch_url")
+        if not unpaginated_fetch_url:
+            return None
+
+        # product types pagination
+        next_page_url_tpl = self.config.discover_product_types.get("next_page_url_tpl")
+        page = self.config.discover_product_types.get("start_page", 1)
+
+        if not next_page_url_tpl:
+            # no pagination
+            return self.discover_product_types_per_page(**kwargs)
+
+        conf_update_dict: Dict[str, Any] = {
+            "providers_config": {},
+            "product_types_config": {},
+        }
+
+        while True:
+            fetch_url = next_page_url_tpl.format(url=unpaginated_fetch_url, page=page)
+
+            conf_update_dict_per_page = self.discover_product_types_per_page(
+                fetch_url=fetch_url, **kwargs
+            )
+
+            if (
+                not conf_update_dict_per_page
+                or not conf_update_dict_per_page.get("providers_config")
+                or conf_update_dict_per_page.items() <= conf_update_dict.items()
+            ):
+                # conf_update_dict_per_page is empty or a subset on existing conf
+                break
+            else:
+                conf_update_dict["providers_config"].update(
+                    conf_update_dict_per_page["providers_config"]
+                )
+                conf_update_dict["product_types_config"].update(
+                    conf_update_dict_per_page["product_types_config"]
+                )
+
+            page += 1
+
+        return conf_update_dict
+
+    def discover_product_types_per_page(
+        self, **kwargs: Any
+    ) -> Optional[Dict[str, Any]]:
+        """Fetch product types list from provider using `discover_product_types` conf
+        using paginated ``kwargs["fetch_url"]``
+
         :returns: configuration dict containing fetched product types information
         """
         try:
             prep = PreparedSearch()
 
-            prep.url = cast(
-                str,
-                self.config.discover_product_types["fetch_url"].format(
-                    **self.config.__dict__
-                ),
-            )
+            # url from discover_product_types() or conf
+            fetch_url: Optional[str] = kwargs.get("fetch_url")
+            if fetch_url is None:
+                if fetch_url := self.config.discover_product_types.get("fetch_url"):
+                    fetch_url = fetch_url.format(**self.config.__dict__)
+                else:
+                    return None
+            prep.url = fetch_url
 
             # get auth if available
             if "auth" in kwargs:
@@ -402,7 +544,14 @@ class QueryStringSearch(Search):
                 "Skipping error while fetching product types for " "{} {} instance:"
             ).format(self.provider, self.__class__.__name__)
 
-            response = QueryStringSearch._request(self, prep)
+            # Query using appropriate method
+            fetch_method = self.config.discover_product_types.get("fetch_method", "GET")
+            fetch_body = self.config.discover_product_types.get("fetch_body", {})
+            if fetch_method == "POST" and isinstance(self, PostJsonSearch):
+                prep.query_params = fetch_body
+                response = self._request(prep)
+            else:
+                response = QueryStringSearch._request(self, prep)
         except (RequestError, KeyError, AttributeError):
             return None
         else:
@@ -414,16 +563,21 @@ class QueryStringSearch(Search):
                 if self.config.discover_product_types["result_type"] == "json":
                     resp_as_json = response.json()
                     # extract results from response json
-                    result = [
-                        match.value
-                        for match in self.config.discover_product_types[
-                            "results_entry"
-                        ].find(resp_as_json)
-                    ]
+                    results_entry = self.config.discover_product_types["results_entry"]
+                    if not isinstance(results_entry, JSONPath):
+                        logger.warning(
+                            f"Could not parse {self.provider} discover_product_types.results_entry"
+                            f" as JSONPath: {results_entry}"
+                        )
+                        return None
+                    result = [match.value for match in results_entry.find(resp_as_json)]
                     if result and isinstance(result[0], list):
                         result = result[0]
 
-                    for product_type_result in result:
+                    def conf_update_from_product_type_result(
+                        product_type_result: Dict[str, Any]
+                    ) -> None:
+                        """Update ``conf_update_dict`` using given product type json response"""
                         # providers_config extraction
                         extracted_mapping = properties_from_json(
                             product_type_result,
@@ -510,6 +664,20 @@ class QueryStringSearch(Search):
                         conf_update_dict["product_types_config"][
                             generic_product_type_id
                         ]["keywords"] = keywords_values_str
+
+                    # runs concurrent requests and aggregate results in conf_update_dict
+                    max_connections = self.config.discover_product_types.get(
+                        "max_connections"
+                    )
+                    with concurrent.futures.ThreadPoolExecutor(
+                        max_workers=max_connections
+                    ) as executor:
+                        futures = (
+                            executor.submit(conf_update_from_product_type_result, r)
+                            for r in result
+                        )
+                        [f.result() for f in concurrent.futures.as_completed(futures)]
+
             except KeyError as e:
                 logger.warning(
                     "Incomplete %s discover_product_types configuration: %s",
@@ -517,6 +685,12 @@ class QueryStringSearch(Search):
                     e,
                 )
                 return None
+            except requests.RequestException as e:
+                logger.debug(
+                    "Could not parse discovered product types response from "
+                    f"{self.provider}, {type(e).__name__}: {e.args}"
+                )
+                return None
         conf_update_dict["product_types_config"] = dict_items_recursive_apply(
             conf_update_dict["product_types_config"],
             lambda k, v: v if v != NOT_AVAILABLE else None,
@@ -538,9 +712,7 @@ class QueryStringSearch(Search):
             self,
             PreparedSearch(
                 url=single_collection_url,
-                info_message="Fetching data for product type product type: {}".format(
-                    product_type
-                ),
+                info_message=f"Fetching data for product type: {product_type}",
                 exception_message="Skipping error while fetching product types for "
                 "{} {} instance:".format(self.provider, self.__class__.__name__),
             ),
@@ -703,9 +875,6 @@ class QueryStringSearch(Search):
                 }
             )
 
-        if product_type is None:
-            raise ValidationError("Required productType is missing")
-
         qp, qs = self.build_query_string(product_type, **keywords)
 
         prep.query_params = qp
@@ -825,7 +994,7 @@ class QueryStringSearch(Search):
             else:
                 next_url = "{}?{}".format(search_endpoint, qs_with_sort)
             urls.append(next_url)
-        return urls, total_results
+        return list(dict.fromkeys(urls)), total_results
 
     def do_search(
         self, prep: PreparedSearch = PreparedSearch(items_per_page=None), **kwargs: Any
@@ -854,8 +1023,8 @@ class QueryStringSearch(Search):
                 search_url
             )
             single_search_prep.exception_message = (
-                "Skipping error while searching for {} {} "
-                "instance:".format(self.provider, self.__class__.__name__)
+                f"Skipping error while searching for {self.provider}"
+                f" {self.__class__.__name__} instance"
             )
             response = self._request(single_search_prep)
             next_page_url_key_path = self.config.pagination.get(
@@ -1114,6 +1283,14 @@ class QueryStringSearch(Search):
             timeout = getattr(self.config, "timeout", HTTP_REQ_TIMEOUT)
             ssl_verify = getattr(self.config, "ssl_verify", True)
 
+            retry_total = getattr(self.config, "retry_total", REQ_RETRY_TOTAL)
+            retry_backoff_factor = getattr(
+                self.config, "retry_backoff_factor", REQ_RETRY_BACKOFF_FACTOR
+            )
+            retry_status_forcelist = getattr(
+                self.config, "retry_status_forcelist", REQ_RETRY_STATUS_FORCELIST
+            )
+
             ssl_ctx = get_ssl_context(ssl_verify)
             # auth if needed
             kwargs: Dict[str, Any] = {}
@@ -1152,7 +1329,16 @@ class QueryStringSearch(Search):
             else:
                 if info_message:
                     logger.info(info_message)
-                response = requests.get(
+
+                session = requests.Session()
+                retries = Retry(
+                    total=retry_total,
+                    backoff_factor=retry_backoff_factor,
+                    status_forcelist=retry_status_forcelist,
+                )
+                session.mount(url, HTTPAdapter(max_retries=retries))
+
+                response = session.get(
                     url,
                     timeout=timeout,
                     headers=USER_AGENT,
@@ -1174,13 +1360,54 @@ class QueryStringSearch(Search):
                     self.__class__.__name__,
                     err_msg,
                 )
-            raise RequestError(str(err))
+            raise RequestError.from_error(err, exception_message) from err
         return response
 
 
 class ODataV4Search(QueryStringSearch):
-    """A specialisation of a QueryStringSearch that does a two step search to retrieve
-    all products metadata"""
+    """A specialisation of a :class:`~eodag.plugins.search.qssearch.QueryStringSearch` that does a two step search to
+    retrieve all products metadata. All configuration parameters of
+    :class:`~eodag.plugins.search.qssearch.QueryStringSearch` are also available for this plugin. In addition, the
+    following parameters can be configured:
+
+    :param provider: provider name
+    :param config: Search plugin configuration:
+
+      * :attr:`~eodag.config.PluginConfig.per_product_metadata_query` (``bool``): should be set to true if the metadata
+        is not given in the search result and a two step search has to be performed; default: false
+      * :attr:`~eodag.config.PluginConfig.metadata_pre_mapping` (:class:`~eodag.config.PluginConfig.MetadataPreMapping`)
+        : a dictionary which can be used to simplify further metadata extraction. For example, going from
+        ``$.Metadata[?(@.id="foo")].value`` to ``$.Metadata.foo.value``. It has the keys:
+
+        * :attr:`~eodag.config.PluginConfig.MetadataPreMapping.metadata_path` (``str``): json path of the metadata entry
+        * :attr:`~eodag.config.PluginConfig.MetadataPreMapping.metadata_path_id` (``str``): key to get the metadata id
+        * :attr:`~eodag.config.PluginConfig.MetadataPreMapping.metadata_path_value` (``str``): key to get the metadata
+          value
+
+      * :attr:`~eodag.config.PluginConfig.free_text_search_operations`: (optional) A tree structure of the form::
+
+          # noqa: E800
+          <search-param>:     # e.g: $search
+              union: # how to join the operations below (e.g: ' AND ' -->
+                  # '(op1 AND op2) AND (op3 OR op4)')
+              wrapper: # a pattern for how each operation will be wrapped
+                      # (e.g: '({})' --> '(op1 AND op2)')
+              operations:     # The operations to build
+              <opname>:     # e.g: AND
+                  - <op1>    # e.g:
+                          # 'sensingStartDate:[{startTimeFromAscendingNode}Z TO *]'
+                  - <op2>    # e.g:
+                      # 'sensingStopDate:[* TO {completionTimeFromAscendingNode}Z]'
+                  ...
+              ...
+          ...
+
+        With the structure above, each operation will become a string of the form:
+        ``(<op1> <opname> <op2>)``, then the operations will be joined together using
+        the union string and finally if the number of operations is greater than 1,
+        they will be wrapped as specified by the wrapper config key.
+
+    """
 
     def __init__(self, provider: str, config: PluginConfig) -> None:
         super(ODataV4Search, self).__init__(provider, config)
@@ -1219,7 +1446,7 @@ class ODataV4Search(QueryStringSearch):
                     raise TimeOutError(exc, timeout=HTTP_REQ_TIMEOUT) from exc
                 except requests.RequestException:
                     logger.exception(
-                        "Skipping error while searching for %s %s instance:",
+                        "Skipping error while searching for %s %s instance",
                         self.provider,
                         self.__class__.__name__,
                     )
@@ -1271,7 +1498,106 @@ class ODataV4Search(QueryStringSearch):
 
 
 class PostJsonSearch(QueryStringSearch):
-    """A specialisation of a QueryStringSearch that uses POST method"""
+    """A specialisation of a :class:`~eodag.plugins.search.qssearch.QueryStringSearch` that uses POST method
+
+    All configuration parameters available for :class:`~eodag.plugins.search.qssearch.QueryStringSearch`
+    are also available for PostJsonSearch. The mappings given in metadata_mapping are used to construct
+    a (json) body for the POST request that is sent to the provider. Due to the fact that we sent a POST request and
+    not a get request, the pagination configuration will look slightly different. It has the
+    following parameters:
+
+    :param provider: provider name
+    :param config: Search plugin configuration:
+
+        * :attr:`~eodag.config.PluginConfig.Pagination.next_page_query_obj` (``str``): The additional parameters
+          needed to add pagination information to the search request. These parameters won't be
+          included in result. This must be a json dict formatted like ``{{"foo":"bar"}}`` because
+          it will be passed to a :meth:`str.format` method before being loaded as json.
+        * :attr:`~eodag.config.PluginConfig.Pagination.total_items_nb_key_path` (``str``):  An XPath or JsonPath
+          leading to the total number of results satisfying a request. This is used for providers
+          which provides the total results metadata along with the result of the query and don't
+          have an endpoint for querying the number of items satisfying a request, or for providers
+          for which the count endpoint returns a json or xml document
+        * :attr:`~eodag.config.PluginConfig.Pagination.max_items_per_page` (``int``): The maximum number of items
+          per page that the provider can handle; default: ``50``
+
+    """
+
+    def _get_default_end_date_from_start_date(
+        self, start_datetime: str, product_type: str
+    ) -> str:
+        default_end_date = self.config.products.get(product_type, {}).get(
+            "_default_end_date", None
+        )
+        if default_end_date:
+            return default_end_date
+        try:
+            start_date = datetime.fromisoformat(start_datetime)
+        except ValueError:
+            start_date = datetime.strptime(start_datetime, "%Y-%m-%dT%H:%M:%SZ")
+        product_type_conf = self.config.products[product_type]
+        if (
+            "metadata_mapping" in product_type_conf
+            and "startTimeFromAscendingNode" in product_type_conf["metadata_mapping"]
+        ):
+            mapping = product_type_conf["metadata_mapping"][
+                "startTimeFromAscendingNode"
+            ]
+            if isinstance(mapping, list) and "year" in mapping[0]:
+                # if date is mapped to year/month/(day), use end_date = start_date to avoid large requests
+                end_date = start_date
+                return end_date.isoformat()
+        return self.get_product_type_cfg_value("missionEndDate", today().isoformat())
+
+    def _check_date_params(self, keywords: Dict[str, Any], product_type: str) -> None:
+        """checks if start and end date are present in the keywords and adds them if not"""
+        if (
+            "startTimeFromAscendingNode"
+            and "completionTimeFromAscendingNode" in keywords
+        ):
+            return
+        # start time given, end time missing
+        if "startTimeFromAscendingNode" in keywords:
+            keywords[
+                "completionTimeFromAscendingNode"
+            ] = self._get_default_end_date_from_start_date(
+                keywords["startTimeFromAscendingNode"], product_type
+            )
+            return
+        product_type_conf = self.config.products[product_type]
+        if (
+            "metadata_mapping" in product_type_conf
+            and "startTimeFromAscendingNode" in product_type_conf["metadata_mapping"]
+        ):
+            mapping = product_type_conf["metadata_mapping"][
+                "startTimeFromAscendingNode"
+            ]
+            if isinstance(mapping, list):
+                # get time parameters (date, year, month, ...) from metadata mapping
+                input_mapping = mapping[0].replace("{{", "").replace("}}", "")
+                time_params = [
+                    values.split(":")[0].strip() for values in input_mapping.split(",")
+                ]
+                time_params = [
+                    tp.replace('"', "").replace("'", "") for tp in time_params
+                ]
+                # if startTime is not given but other time params (e.g. year/month/(day)) are given,
+                # no default date is required
+                in_keywords = True
+                for tp in time_params:
+                    if tp not in keywords:
+                        in_keywords = False
+                if not in_keywords:
+                    keywords[
+                        "startTimeFromAscendingNode"
+                    ] = self.get_product_type_cfg_value(
+                        "missionStartDate", today().isoformat()
+                    )
+                    keywords[
+                        "completionTimeFromAscendingNode"
+                    ] = self._get_default_end_date_from_start_date(
+                        keywords["startTimeFromAscendingNode"], product_type
+                    )
 
     def query(
         self,
@@ -1322,6 +1648,8 @@ class PostJsonSearch(QueryStringSearch):
                     and isinstance(self.config.metadata_mapping[k], list)
                 }
             )
+            if getattr(self.config, "dates_required", False):
+                self._check_date_params(keywords, product_type)
 
             qp, _ = self.build_query_string(product_type, **keywords)
 
@@ -1503,7 +1831,7 @@ class PostJsonSearch(QueryStringSearch):
                     )
 
             urls.append(search_endpoint)
-        return urls, total_results
+        return list(dict.fromkeys(urls)), total_results
 
     def _request(
         self,
@@ -1537,8 +1865,14 @@ class PostJsonSearch(QueryStringSearch):
                 prep.query_params = self.next_page_query_obj
             if info_message:
                 logger.info(info_message)
-            logger.debug("Query parameters: %s" % prep.query_params)
-            logger.debug("Query kwargs: %s" % kwargs)
+            try:
+                logger.debug("Query parameters: %s" % geojson.dumps(prep.query_params))
+            except TypeError:
+                logger.debug("Query parameters: %s" % prep.query_params)
+            try:
+                logger.debug("Query kwargs: %s" % geojson.dumps(kwargs))
+            except TypeError:
+                logger.debug("Query kwargs: %s" % kwargs)
             response = requests.post(
                 url,
                 json=prep.query_params,
@@ -1551,22 +1885,16 @@ class PostJsonSearch(QueryStringSearch):
         except requests.exceptions.Timeout as exc:
             raise TimeOutError(exc, timeout=timeout) from exc
         except (requests.RequestException, URLError) as err:
+            response = locals().get("response", Response())
             # check if error is identified as auth_error in provider conf
             auth_errors = getattr(self.config, "auth_error_code", [None])
             if not isinstance(auth_errors, list):
                 auth_errors = [auth_errors]
-            if (
-                hasattr(err, "response")
-                and err.response is not None
-                and getattr(err.response, "status_code", None)
-                and err.response.status_code in auth_errors
-            ):
+            if response.status_code and response.status_code in auth_errors:
                 raise AuthenticationError(
-                    "HTTP Error {} returned:\n{}\nPlease check your credentials for {}".format(
-                        err.response.status_code,
-                        err.response.text.strip(),
-                        self.provider,
-                    )
+                    f"Please check your credentials for {self.provider}.",
+                    f"HTTP Error {response.status_code} returned.",
+                    response.text.strip(),
                 )
             if exception_message:
                 logger.exception(exception_message)
@@ -1577,21 +1905,23 @@ class PostJsonSearch(QueryStringSearch):
                     self.provider,
                     self.__class__.__name__,
                 )
-            if "response" in locals():
-                logger.debug(response.content)
-            error_text = str(err)
-            if (
-                hasattr(err, "response")
-                and err.response is not None
-                and getattr(err.response, "text", None)
-            ):
-                error_text = err.response.text
-            raise RequestError(error_text) from err
+            logger.debug(response.content or str(err))
+            raise RequestError.from_error(err, exception_message) from err
         return response
 
 
 class StacSearch(PostJsonSearch):
-    """A specialisation of a QueryStringSearch that uses generic STAC configuration"""
+    """A specialisation of :class:`~eodag.plugins.search.qssearch.PostJsonSearch` that uses generic
+    STAC configuration, it therefore has the same configuration parameters (those inherited
+    from :class:`~eodag.plugins.search.qssearch.QueryStringSearch`).
+    For providers using ``StacSearch`` default values are defined for most of the parameters
+    (see ``stac_provider.yml``). If some parameters are different for a specific provider, they
+    have to be overwritten. If certain functionalities are not available, their configuration
+    parameters have to be overwritten with ``null``. E.g. if there is no queryables endpoint,
+    the :attr:`~eodag.config.PluginConfig.DiscoverQueryables.fetch_url` and
+    :attr:`~eodag.config.PluginConfig.DiscoverQueryables.product_type_fetch_url` in the
+    :attr:`~eodag.config.PluginConfig.discover_queryables` config have to be set to ``null``.
+    """
 
     def __init__(self, provider: str, config: PluginConfig) -> None:
         # backup results_entry overwritten by init
@@ -1637,12 +1967,35 @@ class StacSearch(PostJsonSearch):
                        arguments)
         :returns: fetched queryable parameters dict
         """
+        if (
+            not self.config.discover_queryables["fetch_url"]
+            and not self.config.discover_queryables["product_type_fetch_url"]
+        ):
+            logger.info(f"Cannot fetch queryables with {self.provider}")
+            return None
+
         product_type = kwargs.get("productType", None)
         provider_product_type = (
             self.config.products.get(product_type, {}).get("productType", product_type)
             if product_type
             else None
         )
+        if (
+            provider_product_type
+            and not self.config.discover_queryables["product_type_fetch_url"]
+        ):
+            logger.info(
+                f"Cannot fetch queryables for a specific product type with {self.provider}"
+            )
+            return None
+        if (
+            not provider_product_type
+            and not self.config.discover_queryables["fetch_url"]
+        ):
+            logger.info(
+                f"Cannot fetch global queryables with {self.provider}. A product type must be specified"
+            )
+            return None
 
         try:
             unparsed_fetch_url = (
@@ -1650,14 +2003,22 @@ class StacSearch(PostJsonSearch):
                 if provider_product_type
                 else self.config.discover_queryables["fetch_url"]
             )
+            if unparsed_fetch_url is None:
+                return None
 
             fetch_url = unparsed_fetch_url.format(
                 provider_product_type=provider_product_type, **self.config.__dict__
             )
+            auth = (
+                self.auth
+                if hasattr(self, "auth") and isinstance(self.auth, AuthBase)
+                else None
+            )
             response = QueryStringSearch._request(
                 self,
                 PreparedSearch(
                     url=fetch_url,
+                    auth=auth,
                     info_message="Fetching queryables: {}".format(fetch_url),
                     exception_message="Skipping error while fetching queryables for "
                     "{} {} instance:".format(self.provider, self.__class__.__name__),
@@ -1671,11 +2032,15 @@ class StacSearch(PostJsonSearch):
                 resp_as_json = response.json()
 
                 # extract results from response json
-                json_queryables = [
-                    match.value
-                    for match in self.config.discover_queryables["results_entry"].find(
-                        resp_as_json
+                results_entry = self.config.discover_queryables["results_entry"]
+                if not isinstance(results_entry, JSONPath):
+                    logger.warning(
+                        f"Could not parse {self.provider} discover_queryables.results_entry"
+                        f" as JSONPath: {results_entry}"
                     )
+                    return None
+                json_queryables = [
+                    match.value for match in results_entry.find(resp_as_json)
                 ][0]
 
             except KeyError as e:
@@ -1709,3 +2074,18 @@ class StacSearch(PostJsonSearch):
             python_queryables = create_model("m", **field_definitions).model_fields
 
         return model_fields_to_annotated(python_queryables)
+
+
+class PostJsonSearchWithStacQueryables(StacSearch, PostJsonSearch):
+    """A specialisation of a :class:`~eodag.plugins.search.qssearch.PostJsonSearch` that uses
+    generic STAC configuration for queryables (inherited from :class:`~eodag.plugins.search.qssearch.StacSearch`).
+    """
+
+    def __init__(self, provider: str, config: PluginConfig) -> None:
+        PostJsonSearch.__init__(self, provider, config)
+
+    def build_query_string(
+        self, product_type: str, **kwargs: Any
+    ) -> Tuple[Dict[str, Any], str]:
+        """Build The query string using the search parameters"""
+        return PostJsonSearch.build_query_string(self, product_type, **kwargs)