eodag 3.0.0b3__py3-none-any.whl → 3.1.0b1__py3-none-any.whl

eodag/plugins/download/s3rest.py

@@ -62,23 +62,28 @@ logger = logging.getLogger("eodag.download.s3rest")
 
 class S3RestDownload(Download):
     """Http download on S3-like object storage location
-    for example using Mundi REST API (free account)
+
+    For example using Mundi REST API (free account)
     https://mundiwebservices.com/keystoneapi/uploads/documents/CWS-DATA-MUT-087-EN-Mundi_Download_v1.1.pdf#page=13
 
-    Re-use AwsDownload bucket some handling methods
+    Re-use AwsDownload bucket and some handling methods
 
     :param provider: provider name
     :param config: Download plugin configuration:
 
-        * ``config.base_uri`` (str) - default endpoint url
-        * ``config.extract`` (bool) - (optional) extract downloaded archive or not
-        * ``config.auth_error_code`` (int) - (optional) authentication error code
-        * ``config.bucket_path_level`` (int) - (optional) bucket location index in path.split('/')
-        * ``config.order_enabled`` (bool) - (optional) wether order is enabled or not if product is `OFFLINE`
-        * ``config.order_method`` (str) - (optional) HTTP request method, GET (default) or POST
-        * ``config.order_headers`` (dict) - (optional) order request headers
-        * ``config.order_on_response`` (dict) - (optional) edit or add new product properties
-        * ``config.order_status`` (:class:`~eodag.config.PluginConfig.OrderStatus`) - Order status handling
+        * :attr:`~eodag.config.PluginConfig.base_uri` (``str``) (**mandatory**): default endpoint url
+        * :attr:`~eodag.config.PluginConfig.extract` (``bool``): extract downloaded archive or not
+        * :attr:`~eodag.config.PluginConfig.auth_error_code` (``int``): authentication error code
+        * :attr:`~eodag.config.PluginConfig.bucket_path_level` (``int``): bucket location index in ``path.split('/')``
+        * :attr:`~eodag.config.PluginConfig.order_enabled` (``bool``): whether order is enabled
+          or not if product is `OFFLINE`
+        * :attr:`~eodag.config.PluginConfig.order_method` (``str``) HTTP request method, ``GET`` (default) or ``POST``
+        * :attr:`~eodag.config.PluginConfig.order_headers` (``[Dict[str, str]]``): order request headers
+        * :attr:`~eodag.config.PluginConfig.order_on_response` (:class:`~eodag.config.PluginConfig.OrderOnResponse`):
+          a typed dictionary containing the key :attr:`~eodag.config.PluginConfig.OrderOnResponse.metadata_mapping`
+          which can be used to add new product properties based on the data in response to the order request
+        * :attr:`~eodag.config.PluginConfig.order_status` (:class:`~eodag.config.PluginConfig.OrderStatus`):
+          Order status handling
     """
 
     def __init__(self, provider: str, config: PluginConfig) -> None:
@@ -90,8 +95,8 @@ class S3RestDownload(Download):
         product: EOProduct,
         auth: Optional[Union[AuthBase, Dict[str, str]]] = None,
         progress_callback: Optional[ProgressCallback] = None,
-        wait: int = DEFAULT_DOWNLOAD_WAIT,
-        timeout: int = DEFAULT_DOWNLOAD_TIMEOUT,
+        wait: float = DEFAULT_DOWNLOAD_WAIT,
+        timeout: float = DEFAULT_DOWNLOAD_TIMEOUT,
         **kwargs: Unpack[DownloadConf],
     ) -> Optional[str]:
         """Download method for S3 REST API.
@@ -125,9 +130,9 @@ class S3RestDownload(Download):
             and "storageStatus" in product.properties
             and product.properties["storageStatus"] != ONLINE_STATUS
         ):
-            self.http_download_plugin.order_download(product=product, auth=auth)
+            self.http_download_plugin._order(product=product, auth=auth)
 
-        @self._download_retry(product, wait, timeout)
+        @self._order_download_retry(product, wait, timeout)
         def download_request(
             product: EOProduct,
             auth: AuthBase,
@@ -137,9 +142,7 @@ class S3RestDownload(Download):
         ):
             # check order status
             if product.properties.get("orderStatusLink", None):
-                self.http_download_plugin.order_download_status(
-                    product=product, auth=auth
-                )
+                self.http_download_plugin._order_status(product=product, auth=auth)
 
             # get bucket urls
             bucket_name, prefix = get_bucket_name_and_prefix(
@@ -189,12 +192,9 @@ class S3RestDownload(Download):
                     auth_errors = [auth_errors]
                 if err.response and err.response.status_code in auth_errors:
                     raise AuthenticationError(
-                        "HTTP Error %s returned, %s\nPlease check your credentials for %s"
-                        % (
-                            err.response.status_code,
-                            err.response.text.strip(),
-                            self.provider,
-                        )
+                        f"Please check your credentials for {self.provider}.",
+                        f"HTTP Error {err.response.status_code} returned.",
+                        err.response.text.strip(),
                     )
                 # product not available
                 elif (
@@ -225,7 +225,7 @@ class S3RestDownload(Download):
                         self.__class__.__name__,
                         bucket_contents.text,
                     )
-                    raise RequestError(str(err))
+                    raise RequestError.from_error(err) from err
             try:
                 xmldoc = minidom.parseString(bucket_contents.text)
             except ExpatError as err:

eodag/plugins/manager.py

@@ -18,6 +18,7 @@
 from __future__ import annotations
 
 import logging
+import re
 from operator import attrgetter
 from pathlib import Path
 from typing import (
@@ -35,17 +36,28 @@ from typing import (
 
 import pkg_resources
 
-from eodag.config import load_config, merge_configs
+from eodag.config import (
+    AUTH_TOPIC_KEYS,
+    PLUGINS_TOPICS_KEYS,
+    load_config,
+    merge_configs,
+)
 from eodag.plugins.apis.base import Api
 from eodag.plugins.authentication.base import Authentication
 from eodag.plugins.base import EODAGPluginMount
 from eodag.plugins.crunch.base import Crunch
 from eodag.plugins.download.base import Download
 from eodag.plugins.search.base import Search
-from eodag.utils import GENERIC_PRODUCT_TYPE
-from eodag.utils.exceptions import MisconfiguredError, UnsupportedProvider
+from eodag.utils import GENERIC_PRODUCT_TYPE, deepcopy, dict_md5sum
+from eodag.utils.exceptions import (
+    AuthenticationError,
+    MisconfiguredError,
+    UnsupportedProvider,
+)
 
 if TYPE_CHECKING:
+    from requests.auth import AuthBase
+
     from eodag.api.product import EOProduct
     from eodag.config import PluginConfig, ProviderConfig
     from eodag.plugins.base import PluginTopic
@@ -70,7 +82,7 @@ class PluginManager:
                              supported by ``eodag``
     """
 
-    supported_topics = {"search", "download", "crunch", "auth", "api"}
+    supported_topics = set(PLUGINS_TOPICS_KEYS)
 
     product_type_to_provider_config_map: Dict[str, List[ProviderConfig]]
 
@@ -139,7 +151,7 @@ class PluginManager:
             self.providers_config = providers_config
 
         self.build_product_type_to_provider_config_map()
-        self._built_plugins_cache: Dict[Tuple[str, str], Any] = {}
+        self._built_plugins_cache: Dict[Tuple[str, str, str], Any] = {}
 
     def build_product_type_to_provider_config_map(self) -> None:
         """Build mapping conf between product types and providers"""
@@ -249,25 +261,146 @@ class PluginManager:
             )
         return plugin
 
-    def get_auth_plugin(self, provider: str) -> Optional[Authentication]:
+    def get_auth_plugin(
+        self, associated_plugin: PluginTopic, product: Optional[EOProduct] = None
+    ) -> Optional[Authentication]:
+        """Build and return the authentication plugin associated to the given
+        search/download plugin
+
+        .. versionchanged:: v3.0.0
+            ``get_auth_plugin()`` now needs ``associated_plugin`` instead of ``provider``
+            as argument.
+
+        :param associated_plugin: The search/download plugin to which the authentication
+                                  plugin is linked
+        :param product: The product to download. ``None`` for search authentication
+        :returns: The Authentication plugin
+        """
+        # matching url from product to download
+        if product is not None and len(product.assets) > 0:
+            matching_url = next(iter(product.assets.values()))["href"]
+        elif product is not None:
+            matching_url = product.properties.get(
+                "downloadLink"
+            ) or product.properties.get("orderLink")
+        else:
+            # search auth
+            matching_url = getattr(associated_plugin.config, "api_endpoint", None)
+
+        try:
+            auth_plugin = next(
+                self.get_auth_plugins(
+                    associated_plugin.provider,
+                    matching_url=matching_url,
+                    matching_conf=associated_plugin.config,
+                )
+            )
+        except StopIteration:
+            auth_plugin = None
+        return auth_plugin
+
+    def get_auth_plugins(
+        self,
+        provider: str,
+        matching_url: Optional[str] = None,
+        matching_conf: Optional[PluginConfig] = None,
+    ) -> Iterator[Authentication]:
         """Build and return the authentication plugin for the given product_type and
         provider
 
         :param provider: The provider for which to get the authentication plugin
-        :returns: The Authentication plugin for the provider
+        :param matching_url: url to compare with plugin matching_url pattern
+        :param matching_conf: configuration to compare with plugin matching_conf
+        :returns: All the Authentication plugins for the given criteria
         """
-        plugin_conf = self.providers_config[provider]
-        auth: Optional[PluginConfig] = getattr(plugin_conf, "auth", None)
-        if not auth:
-            # We guess the plugin being built is of type Api, therefore no need
-            # for an Auth plugin.
-            return None
-        auth.priority = plugin_conf.priority
-        plugin = cast(
-            Authentication,
-            self._build_plugin(provider, auth, Authentication),
-        )
-        return plugin
+        auth_conf: Optional[PluginConfig] = None
+
+        def _is_auth_plugin_matching(
+            auth_conf: PluginConfig,
+            matching_url: Optional[str],
+            matching_conf: Optional[PluginConfig],
+        ) -> bool:
+            plugin_matching_conf = getattr(auth_conf, "matching_conf", {})
+            if matching_conf:
+                if (
+                    plugin_matching_conf
+                    and matching_conf.__dict__.items() >= plugin_matching_conf.items()
+                ):
+                    # conf matches
+                    return True
+            plugin_matching_url = getattr(auth_conf, "matching_url", None)
+            if matching_url:
+                if plugin_matching_url and re.match(
+                    rf"{plugin_matching_url}", matching_url
+                ):
+                    # url matches
+                    return True
+            # no match
+            return False
+
+        # providers configs with given provider at first
+        sorted_providers_config = deepcopy(self.providers_config)
+        sorted_providers_config = {
+            provider: sorted_providers_config.pop(provider),
+            **sorted_providers_config,
+        }
+
+        for plugin_provider, provider_conf in sorted_providers_config.items():
+            for key in AUTH_TOPIC_KEYS:
+                auth_conf = getattr(provider_conf, key, None)
+                if auth_conf is None:
+                    continue
+                # plugin without configured match criteria: only works for given provider
+                unconfigured_match = (
+                    True
+                    if (
+                        not getattr(auth_conf, "matching_conf", {})
+                        and not getattr(auth_conf, "matching_url", None)
+                        and provider == plugin_provider
+                    )
+                    else False
+                )
+
+                if unconfigured_match or _is_auth_plugin_matching(
+                    auth_conf, matching_url, matching_conf
+                ):
+                    auth_conf.priority = provider_conf.priority
+                    plugin = cast(
+                        Authentication,
+                        self._build_plugin(plugin_provider, auth_conf, Authentication),
+                    )
+                    yield plugin
+                else:
+                    continue
+
+    def get_auth(
+        self,
+        provider: str,
+        matching_url: Optional[str] = None,
+        matching_conf: Optional[PluginConfig] = None,
+    ) -> Optional[Union[AuthBase, Dict[str, str]]]:
+        """Authenticate and return the authenticated object for the first matching
+        authentication plugin
+
+        :param provider: The provider for which to get the authentication plugin
+        :param matching_url: url to compare with plugin matching_url pattern
+        :param matching_conf: configuration to compare with plugin matching_conf
+        :returns: All the Authentication plugins for the given criteria
+        """
+        for auth_plugin in self.get_auth_plugins(provider, matching_url, matching_conf):
+            if auth_plugin and callable(getattr(auth_plugin, "authenticate", None)):
+                try:
+                    auth = auth_plugin.authenticate()
+                    return auth
+                except (AuthenticationError, MisconfiguredError) as e:
+                    logger.debug(f"Could not authenticate on {provider}: {str(e)}")
+                    continue
+            else:
+                logger.debug(
+                    f"Could not authenticate on {provider} using {auth_plugin} plugin"
+                )
+                continue
+        return None
 
     @staticmethod
     def get_crunch_plugin(name: str, **options: Any) -> Crunch:
@@ -304,9 +437,11 @@ class PluginManager:
             # Sort the provider configs, taking into account the new priority order
             provider_configs.sort(key=attrgetter("priority"), reverse=True)
         # Update the priority of already built plugins of the given provider
-        for provider_name, topic_class in self._built_plugins_cache:
+        for provider_name, topic_class, auth_match_md5 in self._built_plugins_cache:
             if provider_name == provider:
-                self._built_plugins_cache[(provider, topic_class)].priority = priority
+                self._built_plugins_cache[
+                    (provider, topic_class, auth_match_md5)
+                ].priority = priority
 
     def _build_plugin(
         self,
@@ -325,8 +460,16 @@ class PluginManager:
                 :class:`~eodag.plugin.authentication.Authentication` or
                 :class:`~eodag.plugin.crunch.Crunch`
         """
+        # md5 hash to helps identifying an auth plugin within several for a given provider
+        # (each has distinct matching settings)
+        auth_match_md5 = dict_md5sum(
+            {
+                "matching_url": getattr(plugin_conf, "matching_url", None),
+                "matching_conf": getattr(plugin_conf, "matching_conf", None),
+            }
+        )
         cached_instance = self._built_plugins_cache.setdefault(
-            (provider, topic_class.__name__), None
+            (provider, topic_class.__name__, auth_match_md5), None
         )
         if cached_instance is not None:
             return cached_instance
@@ -336,5 +479,7 @@ class PluginManager:
         plugin: Union[Api, Search, Download, Authentication, Crunch] = plugin_class(
             provider, plugin_conf
         )
-        self._built_plugins_cache[(provider, topic_class.__name__)] = plugin
+        self._built_plugins_cache[
+            (provider, topic_class.__name__, auth_match_md5)
+        ] = plugin
         return plugin

eodag/plugins/search/base.py

@@ -18,7 +18,7 @@
 from __future__ import annotations
 
 import logging
-from typing import TYPE_CHECKING
+from typing import TYPE_CHECKING, Annotated, get_args
 
 import orjson
 from pydantic.fields import Field, FieldInfo
@@ -31,15 +31,13 @@ from eodag.api.product.metadata_mapping import (
 from eodag.plugins.base import PluginTopic
 from eodag.plugins.search import PreparedSearch
 from eodag.types import model_fields_to_annotated
-from eodag.types.queryables import Queryables
+from eodag.types.queryables import Queryables, QueryablesDict
 from eodag.types.search_args import SortByList
 from eodag.utils import (
     GENERIC_PRODUCT_TYPE,
-    Annotated,
     copy_deepcopy,
     deepcopy,
     format_dict_items,
-    get_args,
     update_nested_dict,
 )
 from eodag.utils.exceptions import ValidationError
@@ -95,9 +93,9 @@ class Search(PluginTopic):
     ) -> Tuple[List[EOProduct], Optional[int]]:
         """Implementation of how the products must be searched goes here.
 
-        This method must return a tuple with (1) a list of EOProduct instances (see eodag.api.product module)
-        which will be processed by a Download plugin (2) and the total number of products matching
-        the search criteria. If ``prep.count`` is False, the second element returned must be ``None``.
+        This method must return a tuple with (1) a list of :class:`~eodag.api.product._product.EOProduct` instances
+        which will be processed by a :class:`~eodag.plugins.download.base.Download` plugin (2) and the total number of
+        products matching the search criteria. If ``prep.count`` is False, the second element returned must be ``None``.
         """
         raise NotImplementedError("A Search plugin must implement a method named query")
 
@@ -108,9 +106,9 @@ class Search(PluginTopic):
     def discover_queryables(
         self, **kwargs: Any
     ) -> Optional[Dict[str, Annotated[Any, FieldInfo]]]:
-        """Fetch queryables list from provider using `discover_queryables` conf
+        """Fetch queryables list from provider using :attr:`~eodag.config.PluginConfig.discover_queryables` conf
 
-        :param kwargs: additional filters for queryables (`productType` and other search
+        :param kwargs: additional filters for queryables (``productType`` and other search
                        arguments)
         :returns: fetched queryable parameters dict
         """
@@ -179,6 +177,27 @@ class Search(PluginTopic):
         else:
             return {}
 
+    def get_product_type_cfg_value(self, key: str, default: Any = None) -> Any:
+        """
+        Get the value of a configuration option specific to the current product type.
+
+        This method retrieves the value of a configuration option from the
+        ``product_type_config`` attribute. If the option is not found, the provided
+        default value is returned.
+
+        :param key: The configuration option key.
+        :type key: str
+        :param default: The default value to be returned if the option is not found (default is None).
+        :type default: Any
+
+        :return: The value of the specified configuration option or the default value.
+        :rtype: Any
+        """
+        product_type_cfg = getattr(self.config, "product_type_config", {})
+        non_none_cfg = {k: v for k, v in product_type_cfg.items() if v}
+
+        return non_none_cfg.get(key, default)
+
     def get_metadata_mapping(
         self, product_type: Optional[str] = None
     ) -> Dict[str, Union[str, List[str]]]:
@@ -194,10 +213,10 @@ class Search(PluginTopic):
         return self.config.metadata_mapping
 
     def get_sort_by_arg(self, kwargs: Dict[str, Any]) -> Optional[SortByList]:
-        """Extract the "sort_by" argument from the kwargs or the provider default sort configuration
+        """Extract the ``sort_by`` argument from the kwargs or the provider default sort configuration
 
         :param kwargs: Search arguments
-        :returns: The "sort_by" argument from the kwargs or the provider default sort configuration
+        :returns: The ``sort_by`` argument from the kwargs or the provider default sort configuration
         """
         # remove "sort_by" from search args if exists because it is not part of metadata mapping,
         # it will complete the query string or body once metadata mapping will be done
@@ -216,16 +235,16 @@ class Search(PluginTopic):
         self, sort_by_arg: SortByList
     ) -> Tuple[str, Dict[str, List[Dict[str, str]]]]:
         """Build the sorting part of the query string or body by transforming
-        the "sort_by" argument into a provider-specific string or dictionnary
+        the ``sort_by`` argument into a provider-specific string or dictionary
 
-        :param sort_by_arg: the "sort_by" argument in EODAG format
-        :returns: The "sort_by" argument in provider-specific format
+        :param sort_by_arg: the ``sort_by`` argument in EODAG format
+        :returns: The ``sort_by`` argument in provider-specific format
         """
         if not hasattr(self.config, "sort"):
             raise ValidationError(f"{self.provider} does not support sorting feature")
         # TODO: remove this code block when search args model validation is embeded
         # remove duplicates
-        sort_by_arg = list(set(sort_by_arg))
+        sort_by_arg = list(dict.fromkeys(sort_by_arg))
 
         sort_by_qs: str = ""
         sort_by_qp: Dict[str, Any] = {}
@@ -306,35 +325,93 @@ class Search(PluginTopic):
                 sort_by_qs += parsed_sort_by_tpl
         return (sort_by_qs, sort_by_qp)
 
+    def _get_product_type_queryables(
+        self, product_type: Optional[str], alias: Optional[str], filters: Dict[str, Any]
+    ) -> Dict[str, Annotated[Any, FieldInfo]]:
+        default_values: Dict[str, Any] = deepcopy(
+            getattr(self.config, "products", {}).get(product_type, {})
+        )
+        default_values.pop("metadata_mapping", None)
+        try:
+            filters["productType"] = product_type
+            return self.discover_queryables(**{**default_values, **filters}) or {}
+        except NotImplementedError:
+            return self.queryables_from_metadata_mapping(product_type, alias)
+
     def list_queryables(
         self,
         filters: Dict[str, Any],
+        available_product_types: List[Any],
+        product_type_configs: Dict[str, Dict[str, Any]],
         product_type: Optional[str] = None,
-    ) -> Dict[str, Annotated[Any, FieldInfo]]:
+        alias: Optional[str] = None,
+    ) -> QueryablesDict:
         """
         Get queryables
 
         :param filters: Additional filters for queryables.
+        :param available_product_types: list of available product types
+        :param product_type_configs: dict containing the product type information for all used product types
         :param product_type: (optional) The product type.
+        :param alias: (optional) alias of the product type
 
         :return: A dictionary containing the queryable properties, associating parameters to their
                 annotated type.
         """
-        default_values: Dict[str, Any] = deepcopy(
-            getattr(self.config, "products", {}).get(product_type, {})
+        additional_info = (
+            "Please select a product type to get the possible values of the parameters!"
+            if not product_type
+            else ""
         )
-        default_values.pop("metadata_mapping", None)
-
-        queryables: Dict[str, Annotated[Any, FieldInfo]] = {}
-        try:
-            queryables = self.discover_queryables(**{**default_values, **filters}) or {}
-        except NotImplementedError:
-            pass
+        if product_type or getattr(self.config, "discover_queryables", {}).get(
+            "fetch_url", ""
+        ):
+            if product_type:
+                self.config.product_type_config = product_type_configs[product_type]
+            queryables = self._get_product_type_queryables(product_type, alias, filters)
+            if getattr(self.config, "discover_queryables", {}).get(
+                "constraints_url", ""
+            ):
+                additional_properties = False
+            else:
+                additional_properties = True
+            return QueryablesDict(
+                additional_properties=additional_properties,
+                additional_information=additional_info,
+                **queryables,
+            )
+        else:
+            all_queryables: Dict[str, Any] = {}
+            for pt in available_product_types:
+                self.config.product_type_config = product_type_configs[pt]
+                pt_queryables = self._get_product_type_queryables(pt, None, filters)
+                # only use key and type because values and defaults will vary between product types
+                pt_queryables_neutral = {
+                    k: Annotated[v.__args__[0], Field(default=None)]
+                    for k, v in pt_queryables.items()
+                }
+                all_queryables.update(pt_queryables_neutral)
+            return QueryablesDict(
+                additional_properties=True,
+                additional_information=additional_info,
+                **all_queryables,
+            )
 
+    def queryables_from_metadata_mapping(
+        self, product_type: Optional[str] = None, alias: Optional[str] = None
+    ) -> Dict[str, Annotated[Any, FieldInfo]]:
+        """
+        Extract queryable parameters from product type metadata mapping.
+        :param product_type: product type id (optional)
+        :param alias: (optional) alias of the product type
+        :returns: dict of annotated queryables
+        """
         metadata_mapping: Dict[str, Any] = deepcopy(
             self.get_metadata_mapping(product_type)
         )
 
+        queryables: Dict[str, Annotated[Any, FieldInfo]] = {}
+
         for param in list(metadata_mapping.keys()):
             if NOT_MAPPED in metadata_mapping[param] or not isinstance(
                 metadata_mapping[param], list
@@ -344,28 +421,18 @@ class Search(PluginTopic):
         eodag_queryables = copy_deepcopy(
             model_fields_to_annotated(Queryables.model_fields)
         )
+        # add default value for product type
+        if alias:
+            eodag_queryables.pop("productType")
+            eodag_queryables["productType"] = Annotated[str, Field(default=alias)]
         for k, v in eodag_queryables.items():
             eodag_queryable_field_info = (
                 get_args(v)[1] if len(get_args(v)) > 1 else None
             )
             if not isinstance(eodag_queryable_field_info, FieldInfo):
                 continue
-            # keep default field info of eodag queryables
-            if k in filters and k in queryables:
-                queryable_field_info = (
-                    get_args(queryables[k])[1]
-                    if len(get_args(queryables[k])) > 1
-                    else None
-                )
-                if not isinstance(queryable_field_info, FieldInfo):
-                    continue
-                queryable_field_info.default = filters[k]
-                continue
-            if k in queryables:
-                continue
             if eodag_queryable_field_info.is_required() or (
                 (eodag_queryable_field_info.alias or k) in metadata_mapping
             ):
                 queryables[k] = v
-
         return queryables