eodag 3.0.0b3__py3-none-any.whl → 3.1.0__py3-none-any.whl

eodag/plugins/download/s3rest.py

@@ -20,7 +20,7 @@ from __future__ import annotations
 import logging
 import os
 import os.path
-from typing import TYPE_CHECKING, Dict, List, Optional, Union
+from typing import TYPE_CHECKING, Optional, Union
 from xml.dom import minidom
 from xml.parsers.expat import ExpatError
 
@@ -54,6 +54,7 @@ from eodag.utils.exceptions import (
 if TYPE_CHECKING:
     from eodag.api.product import EOProduct
     from eodag.config import PluginConfig
+    from eodag.types import S3SessionKwargs
     from eodag.types.download_args import DownloadConf
     from eodag.utils import Unpack
 
@@ -62,23 +63,28 @@ logger = logging.getLogger("eodag.download.s3rest")
 
 class S3RestDownload(Download):
     """Http download on S3-like object storage location
-    for example using Mundi REST API (free account)
+
+    For example using Mundi REST API (free account)
     https://mundiwebservices.com/keystoneapi/uploads/documents/CWS-DATA-MUT-087-EN-Mundi_Download_v1.1.pdf#page=13
 
-    Re-use AwsDownload bucket some handling methods
+    Re-use AwsDownload bucket and some handling methods
 
     :param provider: provider name
     :param config: Download plugin configuration:
 
-        * ``config.base_uri`` (str) - default endpoint url
-        * ``config.extract`` (bool) - (optional) extract downloaded archive or not
-        * ``config.auth_error_code`` (int) - (optional) authentication error code
-        * ``config.bucket_path_level`` (int) - (optional) bucket location index in path.split('/')
-        * ``config.order_enabled`` (bool) - (optional) wether order is enabled or not if product is `OFFLINE`
-        * ``config.order_method`` (str) - (optional) HTTP request method, GET (default) or POST
-        * ``config.order_headers`` (dict) - (optional) order request headers
-        * ``config.order_on_response`` (dict) - (optional) edit or add new product properties
-        * ``config.order_status`` (:class:`~eodag.config.PluginConfig.OrderStatus`) - Order status handling
+        * :attr:`~eodag.config.PluginConfig.base_uri` (``str``) (**mandatory**): default endpoint url
+        * :attr:`~eodag.config.PluginConfig.extract` (``bool``): extract downloaded archive or not
+        * :attr:`~eodag.config.PluginConfig.auth_error_code` (``int``): authentication error code
+        * :attr:`~eodag.config.PluginConfig.bucket_path_level` (``int``): bucket location index in ``path.split('/')``
+        * :attr:`~eodag.config.PluginConfig.order_enabled` (``bool``): whether order is enabled
+          or not if product is `OFFLINE`
+        * :attr:`~eodag.config.PluginConfig.order_method` (``str``) HTTP request method, ``GET`` (default) or ``POST``
+        * :attr:`~eodag.config.PluginConfig.order_headers` (``[dict[str, str]]``): order request headers
+        * :attr:`~eodag.config.PluginConfig.order_on_response` (:class:`~eodag.config.PluginConfig.OrderOnResponse`):
+          a typed dictionary containing the key :attr:`~eodag.config.PluginConfig.OrderOnResponse.metadata_mapping`
+          which can be used to add new product properties based on the data in response to the order request
+        * :attr:`~eodag.config.PluginConfig.order_status` (:class:`~eodag.config.PluginConfig.OrderStatus`):
+          Order status handling
     """
 
     def __init__(self, provider: str, config: PluginConfig) -> None:
@@ -88,10 +94,10 @@ class S3RestDownload(Download):
     def download(
         self,
         product: EOProduct,
-        auth: Optional[Union[AuthBase, Dict[str, str]]] = None,
+        auth: Optional[Union[AuthBase, S3SessionKwargs]] = None,
         progress_callback: Optional[ProgressCallback] = None,
-        wait: int = DEFAULT_DOWNLOAD_WAIT,
-        timeout: int = DEFAULT_DOWNLOAD_TIMEOUT,
+        wait: float = DEFAULT_DOWNLOAD_WAIT,
+        timeout: float = DEFAULT_DOWNLOAD_TIMEOUT,
         **kwargs: Unpack[DownloadConf],
     ) -> Optional[str]:
         """Download method for S3 REST API.
@@ -125,9 +131,9 @@ class S3RestDownload(Download):
             and "storageStatus" in product.properties
             and product.properties["storageStatus"] != ONLINE_STATUS
         ):
-            self.http_download_plugin.order_download(product=product, auth=auth)
+            self.http_download_plugin._order(product=product, auth=auth)
 
-        @self._download_retry(product, wait, timeout)
+        @self._order_download_retry(product, wait, timeout)
         def download_request(
             product: EOProduct,
             auth: AuthBase,
@@ -137,9 +143,7 @@ class S3RestDownload(Download):
         ):
             # check order status
             if product.properties.get("orderStatusLink", None):
-                self.http_download_plugin.order_download_status(
-                    product=product, auth=auth
-                )
+                self.http_download_plugin._order_status(product=product, auth=auth)
 
             # get bucket urls
             bucket_name, prefix = get_bucket_name_and_prefix(
@@ -189,12 +193,9 @@ class S3RestDownload(Download):
                     auth_errors = [auth_errors]
                 if err.response and err.response.status_code in auth_errors:
                     raise AuthenticationError(
-                        "HTTP Error %s returned, %s\nPlease check your credentials for %s"
-                        % (
-                            err.response.status_code,
-                            err.response.text.strip(),
-                            self.provider,
-                        )
+                        f"Please check your credentials for {self.provider}.",
+                        f"HTTP Error {err.response.status_code} returned.",
+                        err.response.text.strip(),
                     )
                 # product not available
                 elif (
@@ -225,7 +226,7 @@ class S3RestDownload(Download):
                         self.__class__.__name__,
                         bucket_contents.text,
                     )
-                    raise RequestError(str(err))
+                    raise RequestError.from_error(err) from err
             try:
                 xmldoc = minidom.parseString(bucket_contents.text)
             except ExpatError as err:
@@ -270,7 +271,7 @@ class S3RestDownload(Download):
                 os.remove(record_filename)
 
             # total size for progress_callback
-            size_list: List[int] = [
+            size_list: list[int] = [
                 int(node.firstChild.nodeValue)  # type: ignore[attr-defined]
                 for node in xmldoc.getElementsByTagName("Size")
                 if node.firstChild is not None

eodag/plugins/manager.py

@@ -18,37 +18,39 @@
 from __future__ import annotations
 
 import logging
+import re
 from operator import attrgetter
 from pathlib import Path
-from typing import (
-    TYPE_CHECKING,
-    Any,
-    Dict,
-    Iterator,
-    List,
-    Optional,
-    Tuple,
-    Type,
-    Union,
-    cast,
-)
+from typing import TYPE_CHECKING, Any, Iterator, Optional, Union, cast
 
 import pkg_resources
 
-from eodag.config import load_config, merge_configs
+from eodag.config import (
+    AUTH_TOPIC_KEYS,
+    PLUGINS_TOPICS_KEYS,
+    load_config,
+    merge_configs,
+)
 from eodag.plugins.apis.base import Api
 from eodag.plugins.authentication.base import Authentication
 from eodag.plugins.base import EODAGPluginMount
 from eodag.plugins.crunch.base import Crunch
 from eodag.plugins.download.base import Download
 from eodag.plugins.search.base import Search
-from eodag.utils import GENERIC_PRODUCT_TYPE
-from eodag.utils.exceptions import MisconfiguredError, UnsupportedProvider
+from eodag.utils import GENERIC_PRODUCT_TYPE, deepcopy, dict_md5sum
+from eodag.utils.exceptions import (
+    AuthenticationError,
+    MisconfiguredError,
+    UnsupportedProvider,
+)
 
 if TYPE_CHECKING:
+    from requests.auth import AuthBase
+
     from eodag.api.product import EOProduct
     from eodag.config import PluginConfig, ProviderConfig
     from eodag.plugins.base import PluginTopic
+    from eodag.types import S3SessionKwargs
 
 
 logger = logging.getLogger("eodag.plugins.manager")
@@ -70,13 +72,13 @@ class PluginManager:
                              supported by ``eodag``
     """
 
-    supported_topics = {"search", "download", "crunch", "auth", "api"}
+    supported_topics = set(PLUGINS_TOPICS_KEYS)
 
-    product_type_to_provider_config_map: Dict[str, List[ProviderConfig]]
+    product_type_to_provider_config_map: dict[str, list[ProviderConfig]]
 
-    skipped_plugins: List[str]
+    skipped_plugins: list[str]
 
-    def __init__(self, providers_config: Dict[str, ProviderConfig]) -> None:
+    def __init__(self, providers_config: dict[str, ProviderConfig]) -> None:
         self.skipped_plugins = []
         self.providers_config = providers_config
         # Load all the plugins. This will make all plugin classes of a particular
@@ -132,14 +134,14 @@ class PluginManager:
         self.rebuild()
 
     def rebuild(
-        self, providers_config: Optional[Dict[str, ProviderConfig]] = None
+        self, providers_config: Optional[dict[str, ProviderConfig]] = None
     ) -> None:
         """(Re)Build plugin manager mapping and cache"""
         if providers_config is not None:
             self.providers_config = providers_config
 
         self.build_product_type_to_provider_config_map()
-        self._built_plugins_cache: Dict[Tuple[str, str], Any] = {}
+        self._built_plugins_cache: dict[tuple[str, str, str], Any] = {}
 
     def build_product_type_to_provider_config_map(self) -> None:
         """Build mapping conf between product types and providers"""
@@ -199,7 +201,7 @@ class PluginManager:
                 )
             return plugin
 
-        configs: Optional[List[ProviderConfig]]
+        configs: Optional[list[ProviderConfig]]
         if product_type:
             configs = self.product_type_to_provider_config_map.get(product_type)
             if not configs:
@@ -249,25 +251,146 @@ class PluginManager:
             )
         return plugin
 
-    def get_auth_plugin(self, provider: str) -> Optional[Authentication]:
+    def get_auth_plugin(
+        self, associated_plugin: PluginTopic, product: Optional[EOProduct] = None
+    ) -> Optional[Authentication]:
+        """Build and return the authentication plugin associated to the given
+        search/download plugin
+
+        .. versionchanged:: v3.0.0
+            ``get_auth_plugin()`` now needs ``associated_plugin`` instead of ``provider``
+            as argument.
+
+        :param associated_plugin: The search/download plugin to which the authentication
+                                  plugin is linked
+        :param product: The product to download. ``None`` for search authentication
+        :returns: The Authentication plugin
+        """
+        # matching url from product to download
+        if product is not None and len(product.assets) > 0:
+            matching_url = next(iter(product.assets.values()))["href"]
+        elif product is not None:
+            matching_url = product.properties.get(
+                "downloadLink"
+            ) or product.properties.get("orderLink")
+        else:
+            # search auth
+            matching_url = getattr(associated_plugin.config, "api_endpoint", None)
+
+        try:
+            auth_plugin = next(
+                self.get_auth_plugins(
+                    associated_plugin.provider,
+                    matching_url=matching_url,
+                    matching_conf=associated_plugin.config,
+                )
+            )
+        except StopIteration:
+            auth_plugin = None
+        return auth_plugin
+
+    def get_auth_plugins(
+        self,
+        provider: str,
+        matching_url: Optional[str] = None,
+        matching_conf: Optional[PluginConfig] = None,
+    ) -> Iterator[Authentication]:
         """Build and return the authentication plugin for the given product_type and
         provider
 
         :param provider: The provider for which to get the authentication plugin
-        :returns: The Authentication plugin for the provider
+        :param matching_url: url to compare with plugin matching_url pattern
+        :param matching_conf: configuration to compare with plugin matching_conf
+        :returns: All the Authentication plugins for the given criteria
         """
-        plugin_conf = self.providers_config[provider]
-        auth: Optional[PluginConfig] = getattr(plugin_conf, "auth", None)
-        if not auth:
-            # We guess the plugin being built is of type Api, therefore no need
-            # for an Auth plugin.
-            return None
-        auth.priority = plugin_conf.priority
-        plugin = cast(
-            Authentication,
-            self._build_plugin(provider, auth, Authentication),
-        )
-        return plugin
+        auth_conf: Optional[PluginConfig] = None
+
+        def _is_auth_plugin_matching(
+            auth_conf: PluginConfig,
+            matching_url: Optional[str],
+            matching_conf: Optional[PluginConfig],
+        ) -> bool:
+            plugin_matching_conf = getattr(auth_conf, "matching_conf", {})
+            if matching_conf:
+                if (
+                    plugin_matching_conf
+                    and matching_conf.__dict__.items() >= plugin_matching_conf.items()
+                ):
+                    # conf matches
+                    return True
+            plugin_matching_url = getattr(auth_conf, "matching_url", None)
+            if matching_url:
+                if plugin_matching_url and re.match(
+                    rf"{plugin_matching_url}", matching_url
+                ):
+                    # url matches
+                    return True
+            # no match
+            return False
+
+        # providers configs with given provider at first
+        sorted_providers_config = deepcopy(self.providers_config)
+        sorted_providers_config = {
+            provider: sorted_providers_config.pop(provider),
+            **sorted_providers_config,
+        }
+
+        for plugin_provider, provider_conf in sorted_providers_config.items():
+            for key in AUTH_TOPIC_KEYS:
+                auth_conf = getattr(provider_conf, key, None)
+                if auth_conf is None:
+                    continue
+                # plugin without configured match criteria: only works for given provider
+                unconfigured_match = (
+                    True
+                    if (
+                        not getattr(auth_conf, "matching_conf", {})
+                        and not getattr(auth_conf, "matching_url", None)
+                        and provider == plugin_provider
+                    )
+                    else False
+                )
+
+                if unconfigured_match or _is_auth_plugin_matching(
+                    auth_conf, matching_url, matching_conf
+                ):
+                    auth_conf.priority = provider_conf.priority
+                    plugin = cast(
+                        Authentication,
+                        self._build_plugin(plugin_provider, auth_conf, Authentication),
+                    )
+                    yield plugin
+                else:
+                    continue
+
+    def get_auth(
+        self,
+        provider: str,
+        matching_url: Optional[str] = None,
+        matching_conf: Optional[PluginConfig] = None,
+    ) -> Optional[Union[AuthBase, S3SessionKwargs]]:
+        """Authenticate and return the authenticated object for the first matching
+        authentication plugin
+
+        :param provider: The provider for which to get the authentication plugin
+        :param matching_url: url to compare with plugin matching_url pattern
+        :param matching_conf: configuration to compare with plugin matching_conf
+        :returns: All the Authentication plugins for the given criteria
+        """
+        for auth_plugin in self.get_auth_plugins(provider, matching_url, matching_conf):
+            if auth_plugin and callable(getattr(auth_plugin, "authenticate", None)):
+                try:
+                    auth = auth_plugin.authenticate()
+                    return auth
+                except (AuthenticationError, MisconfiguredError) as e:
+                    logger.debug(f"Could not authenticate on {provider}: {str(e)}")
+                    continue
+            else:
+                logger.debug(
+                    f"Could not authenticate on {provider} using {auth_plugin} plugin"
+                )
+                continue
+        return None
 
     @staticmethod
     def get_crunch_plugin(name: str, **options: Any) -> Crunch:
@@ -304,15 +427,17 @@ class PluginManager:
             # Sort the provider configs, taking into account the new priority order
             provider_configs.sort(key=attrgetter("priority"), reverse=True)
         # Update the priority of already built plugins of the given provider
-        for provider_name, topic_class in self._built_plugins_cache:
+        for provider_name, topic_class, auth_match_md5 in self._built_plugins_cache:
             if provider_name == provider:
-                self._built_plugins_cache[(provider, topic_class)].priority = priority
+                self._built_plugins_cache[
+                    (provider, topic_class, auth_match_md5)
+                ].priority = priority
 
     def _build_plugin(
         self,
         provider: str,
         plugin_conf: PluginConfig,
-        topic_class: Type[PluginTopic],
+        topic_class: type[PluginTopic],
     ) -> Union[Api, Search, Download, Authentication, Crunch]:
         """Build the plugin of the given topic with the given plugin configuration and
         registered as the given provider
@@ -325,8 +450,16 @@ class PluginManager:
                 :class:`~eodag.plugin.authentication.Authentication` or
                 :class:`~eodag.plugin.crunch.Crunch`
         """
+        # md5 hash to helps identifying an auth plugin within several for a given provider
+        # (each has distinct matching settings)
+        auth_match_md5 = dict_md5sum(
+            {
+                "matching_url": getattr(plugin_conf, "matching_url", None),
+                "matching_conf": getattr(plugin_conf, "matching_conf", None),
+            }
+        )
         cached_instance = self._built_plugins_cache.setdefault(
-            (provider, topic_class.__name__), None
+            (provider, topic_class.__name__, auth_match_md5), None
         )
         if cached_instance is not None:
             return cached_instance
@@ -336,5 +469,7 @@ class PluginManager:
         plugin: Union[Api, Search, Download, Authentication, Crunch] = plugin_class(
             provider, plugin_conf
         )
-        self._built_plugins_cache[(provider, topic_class.__name__)] = plugin
+        self._built_plugins_cache[
+            (provider, topic_class.__name__, auth_match_md5)
+        ] = plugin
         return plugin

eodag/plugins/search/__init__.py

@@ -24,11 +24,12 @@ from typing import TYPE_CHECKING
 from eodag.utils import DEFAULT_ITEMS_PER_PAGE, DEFAULT_PAGE
 
 if TYPE_CHECKING:
-    from typing import Any, Dict, List, Optional, Union
+    from typing import Any, Optional, Union
 
     from requests.auth import AuthBase
 
     from eodag.plugins.authentication.base import Authentication
+    from eodag.types import S3SessionKwargs
 
 
 @dataclass
@@ -38,7 +39,7 @@ class PreparedSearch:
     product_type: Optional[str] = None
     page: Optional[int] = DEFAULT_PAGE
     items_per_page: Optional[int] = DEFAULT_ITEMS_PER_PAGE
-    auth: Optional[Union[AuthBase, Dict[str, str]]] = None
+    auth: Optional[Union[AuthBase, S3SessionKwargs]] = None
     auth_plugin: Optional[Authentication] = None
     count: bool = True
     url: Optional[str] = None
@@ -46,9 +47,9 @@ class PreparedSearch:
     exception_message: Optional[str] = None
 
     need_count: bool = field(init=False, repr=False)
-    query_params: Dict[str, Any] = field(init=False, repr=False)
+    query_params: dict[str, Any] = field(init=False, repr=False)
     query_string: str = field(init=False, repr=False)
-    search_urls: List[str] = field(init=False, repr=False)
-    product_type_def_params: Dict[str, Any] = field(init=False, repr=False)
+    search_urls: list[str] = field(init=False, repr=False)
+    product_type_def_params: dict[str, Any] = field(init=False, repr=False)
     total_items_nb: int = field(init=False, repr=False)
     sort_by_qs: str = field(init=False, repr=False)