eodag 3.0.0b2__py3-none-any.whl → 3.0.1__py3-none-any.whl

eodag/plugins/download/s3rest.py

@@ -20,7 +20,7 @@ from __future__ import annotations
 import logging
 import os
 import os.path
-from typing import TYPE_CHECKING, Dict, Optional, Union
+from typing import TYPE_CHECKING, Dict, List, Optional, Union
 from xml.dom import minidom
 from xml.parsers.expat import ExpatError
 
@@ -62,26 +62,28 @@ logger = logging.getLogger("eodag.download.s3rest")
 
 class S3RestDownload(Download):
     """Http download on S3-like object storage location
-    for example using Mundi REST API (free account)
+
+    For example using Mundi REST API (free account)
     https://mundiwebservices.com/keystoneapi/uploads/documents/CWS-DATA-MUT-087-EN-Mundi_Download_v1.1.pdf#page=13
 
-    Re-use AwsDownload bucket some handling methods
+    Re-use AwsDownload bucket and some handling methods
 
     :param provider: provider name
-    :type provider: str
     :param config: Download plugin configuration:
 
-        * ``config.base_uri`` (str) - default endpoint url
-        * ``config.extract`` (bool) - (optional) extract downloaded archive or not
-        * ``config.auth_error_code`` (int) - (optional) authentication error code
-        * ``config.bucket_path_level`` (int) - (optional) bucket location index in path.split('/')
-        * ``config.order_enabled`` (bool) - (optional) wether order is enabled or not if product is `OFFLINE`
-        * ``config.order_method`` (str) - (optional) HTTP request method, GET (default) or POST
-        * ``config.order_headers`` (dict) - (optional) order request headers
-        * ``config.order_on_response`` (dict) - (optional) edit or add new product properties
-        * ``config.order_status`` (:class:`~eodag.config.PluginConfig.OrderStatus`) - Order status handling
-
-    :type config: :class:`~eodag.config.PluginConfig`
+        * :attr:`~eodag.config.PluginConfig.base_uri` (``str``) (**mandatory**): default endpoint url
+        * :attr:`~eodag.config.PluginConfig.extract` (``bool``): extract downloaded archive or not
+        * :attr:`~eodag.config.PluginConfig.auth_error_code` (``int``): authentication error code
+        * :attr:`~eodag.config.PluginConfig.bucket_path_level` (``int``): bucket location index in ``path.split('/')``
+        * :attr:`~eodag.config.PluginConfig.order_enabled` (``bool``): whether order is enabled
+          or not if product is `OFFLINE`
+        * :attr:`~eodag.config.PluginConfig.order_method` (``str``) HTTP request method, ``GET`` (default) or ``POST``
+        * :attr:`~eodag.config.PluginConfig.order_headers` (``[Dict[str, str]]``): order request headers
+        * :attr:`~eodag.config.PluginConfig.order_on_response` (:class:`~eodag.config.PluginConfig.OrderOnResponse`):
+          a typed dictionary containing the key :attr:`~eodag.config.PluginConfig.OrderOnResponse.metadata_mapping`
+          which can be used to add new product properties based on the data in response to the order request
+        * :attr:`~eodag.config.PluginConfig.order_status` (:class:`~eodag.config.PluginConfig.OrderStatus`):
+          Order status handling
     """
 
     def __init__(self, provider: str, config: PluginConfig) -> None:
@@ -100,22 +102,17 @@ class S3RestDownload(Download):
         """Download method for S3 REST API.
 
         :param product: The EO product to download
-        :type product: :class:`~eodag.api.product._product.EOProduct`
         :param auth: (optional) authenticated object
-        :type auth: Optional[Union[AuthBase, Dict[str, str]]]
         :param progress_callback: (optional) A method or a callable object
                                   which takes a current size and a maximum
                                   size as inputs and handle progress bar
                                   creation and update to give the user a
                                   feedback on the download progress
-        :type progress_callback: :class:`~eodag.utils.ProgressCallback` or None
-        :param kwargs: `outputs_prefix` (str), `extract` (bool), `delete_archive` (bool)
+        :param kwargs: `output_dir` (str), `extract` (bool), `delete_archive` (bool)
                         and `dl_url_params` (dict) can be provided as additional kwargs
                         and will override any other values defined in a configuration
                         file or with environment variables.
-        :type kwargs: Union[str, bool, dict]
         :returns: The absolute path to the downloaded product in the local filesystem
-        :rtype: str
         """
         if auth is not None and not isinstance(auth, AuthBase):
             raise MisconfiguredError(f"Incompatible auth plugin: {type(auth)}")
@@ -133,7 +130,7 @@ class S3RestDownload(Download):
             and "storageStatus" in product.properties
             and product.properties["storageStatus"] != ONLINE_STATUS
         ):
-            self.http_download_plugin.orderDownload(product=product, auth=auth)
+            self.http_download_plugin.order_download(product=product, auth=auth)
 
         @self._download_retry(product, wait, timeout)
         def download_request(
@@ -145,7 +142,7 @@ class S3RestDownload(Download):
         ):
             # check order status
             if product.properties.get("orderStatusLink", None):
-                self.http_download_plugin.orderDownloadStatus(
+                self.http_download_plugin.order_download_status(
                     product=product, auth=auth
                 )
 
@@ -153,6 +150,8 @@ class S3RestDownload(Download):
             bucket_name, prefix = get_bucket_name_and_prefix(
                 url=product.location, bucket_path_level=self.config.bucket_path_level
             )
+            if prefix is None:
+                raise DownloadError(f"Could not extract prefix from {product.location}")
 
             if (
                 bucket_name is None
@@ -195,12 +194,9 @@ class S3RestDownload(Download):
                     auth_errors = [auth_errors]
                 if err.response and err.response.status_code in auth_errors:
                     raise AuthenticationError(
-                        "HTTP Error %s returned, %s\nPlease check your credentials for %s"
-                        % (
-                            err.response.status_code,
-                            err.response.text.strip(),
-                            self.provider,
-                        )
+                        f"Please check your credentials for {self.provider}.",
+                        f"HTTP Error {err.response.status_code} returned.",
+                        err.response.text.strip(),
                     )
                 # product not available
                 elif (
@@ -231,7 +227,7 @@ class S3RestDownload(Download):
                         self.__class__.__name__,
                         bucket_contents.text,
                     )
-                    raise RequestError(str(err))
+                    raise RequestError.from_error(err) from err
             try:
                 xmldoc = minidom.parseString(bucket_contents.text)
             except ExpatError as err:
@@ -243,14 +239,12 @@ class S3RestDownload(Download):
                 logger.warning("Could not load any content from %s", nodes_list_url)
 
             # destination product path
-            outputs_prefix = (
-                kwargs.pop("outputs_prefix", None) or self.config.outputs_prefix
-            )
-            abs_outputs_prefix = os.path.abspath(outputs_prefix)
-            product_local_path = os.path.join(abs_outputs_prefix, prefix.split("/")[-1])
+            output_dir = kwargs.pop("output_dir", None) or self.config.output_dir
+            abs_output_dir = os.path.abspath(output_dir)
+            product_local_path = os.path.join(abs_output_dir, prefix.split("/")[-1])
 
             # .downloaded cache record directory
-            download_records_dir = os.path.join(abs_outputs_prefix, ".downloaded")
+            download_records_dir = os.path.join(abs_output_dir, ".downloaded")
             try:
                 os.makedirs(download_records_dir)
             except OSError as exc:
@@ -278,18 +272,18 @@ class S3RestDownload(Download):
                 os.remove(record_filename)
 
             # total size for progress_callback
-            total_size = sum(
-                [
-                    int(node.firstChild.nodeValue)
-                    for node in xmldoc.getElementsByTagName("Size")
-                ]
-            )
+            size_list: List[int] = [
+                int(node.firstChild.nodeValue)  # type: ignore[attr-defined]
+                for node in xmldoc.getElementsByTagName("Size")
+                if node.firstChild is not None
+            ]
+            total_size = sum(size_list)
             progress_callback.reset(total=total_size)
 
             # download each node key
             for node_xml in nodes_xml_list:
                 node_key = unquote(
-                    node_xml.getElementsByTagName("Key")[0].firstChild.nodeValue
+                    node_xml.getElementsByTagName("Key")[0].firstChild.nodeValue  # type: ignore[union-attr]
                 )
                 # As "Key", "Size" and "ETag" (md5 hash) can also be retrieved from node_xml
                 node_url = urljoin(bucket_url.strip("/") + "/", node_key.strip("/"))

eodag/plugins/manager.py

@@ -18,6 +18,7 @@
 from __future__ import annotations
 
 import logging
+import re
 from operator import attrgetter
 from pathlib import Path
 from typing import (
@@ -35,17 +36,28 @@ from typing import (
 
 import pkg_resources
 
-from eodag.config import load_config, merge_configs
+from eodag.config import (
+    AUTH_TOPIC_KEYS,
+    PLUGINS_TOPICS_KEYS,
+    load_config,
+    merge_configs,
+)
 from eodag.plugins.apis.base import Api
 from eodag.plugins.authentication.base import Authentication
 from eodag.plugins.base import EODAGPluginMount
 from eodag.plugins.crunch.base import Crunch
 from eodag.plugins.download.base import Download
 from eodag.plugins.search.base import Search
-from eodag.utils import GENERIC_PRODUCT_TYPE
-from eodag.utils.exceptions import MisconfiguredError, UnsupportedProvider
+from eodag.utils import GENERIC_PRODUCT_TYPE, deepcopy, dict_md5sum
+from eodag.utils.exceptions import (
+    AuthenticationError,
+    MisconfiguredError,
+    UnsupportedProvider,
+)
 
 if TYPE_CHECKING:
+    from requests.auth import AuthBase
+
     from eodag.api.product import EOProduct
     from eodag.config import PluginConfig, ProviderConfig
     from eodag.plugins.base import PluginTopic
@@ -68,10 +80,9 @@ class PluginManager:
 
     :param providers_config: The configuration with all information about the providers
                              supported by ``eodag``
-    :type providers_config: dict[str, :class:`~eodag.config.ProviderConfig`]
     """
 
-    supported_topics = {"search", "download", "crunch", "auth", "api"}
+    supported_topics = set(PLUGINS_TOPICS_KEYS)
 
     product_type_to_provider_config_map: Dict[str, List[ProviderConfig]]
 
@@ -111,7 +122,11 @@ class PluginManager:
                         "Check that the plugin module (%s) is importable",
                         entry_point.module_name,
                     )
-                if entry_point.dist and entry_point.dist.key != "eodag":
+                if (
+                    entry_point.dist
+                    and entry_point.dist.key != "eodag"
+                    and entry_point.dist.location is not None
+                ):
                     # use plugin providers if any
                     plugin_providers_config_path = [
                         str(x)
@@ -136,7 +151,7 @@ class PluginManager:
             self.providers_config = providers_config
 
         self.build_product_type_to_provider_config_map()
-        self._built_plugins_cache: Dict[Tuple[str, str], Any] = {}
+        self._built_plugins_cache: Dict[Tuple[str, str, str], Any] = {}
 
     def build_product_type_to_provider_config_map(self) -> None:
         """Build mapping conf between product types and providers"""
@@ -172,13 +187,10 @@ class PluginManager:
 
         :param product_type: (optional) The product type that the constructed plugins
                              must support
-        :type product_type: str
         :param provider: (optional) The provider or the provider group on which to get
             the search plugins
-        :type provider: str
         :returns: All the plugins supporting the product type, one by one (a generator
                   object)
-        :rtype: types.GeneratorType(:class:`~eodag.plugins.search.Search`
             or :class:`~eodag.plugins.download.Api`)
         :raises: :class:`~eodag.utils.exceptions.UnsupportedProvider`
         """
@@ -230,9 +242,7 @@ class PluginManager:
         product.
 
         :param product: The product to get a download plugin for
-        :type product: :class:`~eodag.api.product._product.EOProduct`
         :returns: The download plugin capable of downloading the product
-        :rtype: :class:`~eodag.plugins.download.Download` or :class:`~eodag.plugins.download.Api`
         """
         plugin_conf = self.providers_config[product.provider]
         if download := getattr(plugin_conf, "download", None):
@@ -251,27 +261,146 @@ class PluginManager:
             )
         return plugin
 
-    def get_auth_plugin(self, provider: str) -> Optional[Authentication]:
+    def get_auth_plugin(
+        self, associated_plugin: PluginTopic, product: Optional[EOProduct] = None
+    ) -> Optional[Authentication]:
+        """Build and return the authentication plugin associated to the given
+        search/download plugin
+
+        .. versionchanged:: v3.0.0
+            ``get_auth_plugin()`` now needs ``associated_plugin`` instead of ``provider``
+            as argument.
+
+        :param associated_plugin: The search/download plugin to which the authentication
+                                  plugin is linked
+        :param product: The product to download. ``None`` for search authentication
+        :returns: The Authentication plugin
+        """
+        # matching url from product to download
+        if product is not None and len(product.assets) > 0:
+            matching_url = next(iter(product.assets.values()))["href"]
+        elif product is not None:
+            matching_url = product.properties.get(
+                "downloadLink"
+            ) or product.properties.get("orderLink")
+        else:
+            # search auth
+            matching_url = getattr(associated_plugin.config, "api_endpoint", None)
+
+        try:
+            auth_plugin = next(
+                self.get_auth_plugins(
+                    associated_plugin.provider,
+                    matching_url=matching_url,
+                    matching_conf=associated_plugin.config,
+                )
+            )
+        except StopIteration:
+            auth_plugin = None
+        return auth_plugin
+
+    def get_auth_plugins(
+        self,
+        provider: str,
+        matching_url: Optional[str] = None,
+        matching_conf: Optional[PluginConfig] = None,
+    ) -> Iterator[Authentication]:
         """Build and return the authentication plugin for the given product_type and
         provider
 
         :param provider: The provider for which to get the authentication plugin
-        :type provider: str
-        :returns: The Authentication plugin for the provider
-        :rtype: :class:`~eodag.plugins.authentication.Authentication`
+        :param matching_url: url to compare with plugin matching_url pattern
+        :param matching_conf: configuration to compare with plugin matching_conf
+        :returns: All the Authentication plugins for the given criteria
         """
-        plugin_conf = self.providers_config[provider]
-        auth: Optional[PluginConfig] = getattr(plugin_conf, "auth", None)
-        if not auth:
-            # We guess the plugin being built is of type Api, therefore no need
-            # for an Auth plugin.
-            return None
-        auth.priority = plugin_conf.priority
-        plugin = cast(
-            Authentication,
-            self._build_plugin(provider, auth, Authentication),
-        )
-        return plugin
+        auth_conf: Optional[PluginConfig] = None
+
+        def _is_auth_plugin_matching(
+            auth_conf: PluginConfig,
+            matching_url: Optional[str],
+            matching_conf: Optional[PluginConfig],
+        ) -> bool:
+            plugin_matching_conf = getattr(auth_conf, "matching_conf", {})
+            if matching_conf:
+                if (
+                    plugin_matching_conf
+                    and matching_conf.__dict__.items() >= plugin_matching_conf.items()
+                ):
+                    # conf matches
+                    return True
+            plugin_matching_url = getattr(auth_conf, "matching_url", None)
+            if matching_url:
+                if plugin_matching_url and re.match(
+                    rf"{plugin_matching_url}", matching_url
+                ):
+                    # url matches
+                    return True
+            # no match
+            return False
+
+        # providers configs with given provider at first
+        sorted_providers_config = deepcopy(self.providers_config)
+        sorted_providers_config = {
+            provider: sorted_providers_config.pop(provider),
+            **sorted_providers_config,
+        }
+
+        for plugin_provider, provider_conf in sorted_providers_config.items():
+            for key in AUTH_TOPIC_KEYS:
+                auth_conf = getattr(provider_conf, key, None)
+                if auth_conf is None:
+                    continue
+                # plugin without configured match criteria: only works for given provider
+                unconfigured_match = (
+                    True
+                    if (
+                        not getattr(auth_conf, "matching_conf", {})
+                        and not getattr(auth_conf, "matching_url", None)
+                        and provider == plugin_provider
+                    )
+                    else False
+                )
+
+                if unconfigured_match or _is_auth_plugin_matching(
+                    auth_conf, matching_url, matching_conf
+                ):
+                    auth_conf.priority = provider_conf.priority
+                    plugin = cast(
+                        Authentication,
+                        self._build_plugin(plugin_provider, auth_conf, Authentication),
+                    )
+                    yield plugin
+                else:
+                    continue
+
+    def get_auth(
+        self,
+        provider: str,
+        matching_url: Optional[str] = None,
+        matching_conf: Optional[PluginConfig] = None,
+    ) -> Optional[Union[AuthBase, Dict[str, str]]]:
+        """Authenticate and return the authenticated object for the first matching
+        authentication plugin
+
+        :param provider: The provider for which to get the authentication plugin
+        :param matching_url: url to compare with plugin matching_url pattern
+        :param matching_conf: configuration to compare with plugin matching_conf
+        :returns: All the Authentication plugins for the given criteria
+        """
+        for auth_plugin in self.get_auth_plugins(provider, matching_url, matching_conf):
+            if auth_plugin and callable(getattr(auth_plugin, "authenticate", None)):
+                try:
+                    auth = auth_plugin.authenticate()
+                    return auth
+                except (AuthenticationError, MisconfiguredError) as e:
+                    logger.debug(f"Could not authenticate on {provider}: {str(e)}")
+                    continue
+            else:
+                logger.debug(
+                    f"Could not authenticate on {provider} using {auth_plugin} plugin"
+                )
+                continue
+        return None
 
     @staticmethod
     def get_crunch_plugin(name: str, **options: Any) -> Crunch:
@@ -279,11 +408,8 @@ class PluginManager:
         it with the `options`
 
         :param name: The name of the Crunch plugin to instantiate
-        :type name: str
         :param options: The configuration parameters of the cruncher
-        :type options: dict
         :returns: The cruncher named `name`
-        :rtype: :class:`~eodag.plugins.crunch.Crunch`
         """
         klass = Crunch.get_plugin_by_class_name(name)
         return klass(options)
@@ -297,9 +423,7 @@ class PluginManager:
         """Set the priority of the given provider
 
         :param provider: The provider which is assigned the priority
-        :type provider: str
         :param priority: The priority to assign to the provider
-        :type priority: int
         """
         # Update the priority in the configurations so that it is taken into account
         # when a plugin of this provider is latterly built
@@ -313,9 +437,11 @@ class PluginManager:
             # Sort the provider configs, taking into account the new priority order
             provider_configs.sort(key=attrgetter("priority"), reverse=True)
         # Update the priority of already built plugins of the given provider
-        for provider_name, topic_class in self._built_plugins_cache:
+        for provider_name, topic_class, auth_match_md5 in self._built_plugins_cache:
             if provider_name == provider:
-                self._built_plugins_cache[(provider, topic_class)].priority = priority
+                self._built_plugins_cache[
+                    (provider, topic_class, auth_match_md5)
+                ].priority = priority
 
     def _build_plugin(
         self,
@@ -327,19 +453,23 @@ class PluginManager:
         registered as the given provider
 
         :param provider: The provider for which to build the plugin
-        :type provider: str
         :param plugin_conf: The configuration of the plugin to be built
-        :type plugin_conf: :class:`~eodag.config.PluginConfig`
         :param topic_class: The type of plugin to build
-        :type topic_class: :class:`~eodag.plugin.base.PluginTopic`
         :returns: The built plugin
-        :rtype: :class:`~eodag.plugin.search.Search` or
                 :class:`~eodag.plugin.download.Download` or
                 :class:`~eodag.plugin.authentication.Authentication` or
                 :class:`~eodag.plugin.crunch.Crunch`
         """
+        # md5 hash to helps identifying an auth plugin within several for a given provider
+        # (each has distinct matching settings)
+        auth_match_md5 = dict_md5sum(
+            {
+                "matching_url": getattr(plugin_conf, "matching_url", None),
+                "matching_conf": getattr(plugin_conf, "matching_conf", None),
+            }
+        )
         cached_instance = self._built_plugins_cache.setdefault(
-            (provider, topic_class.__name__), None
+            (provider, topic_class.__name__, auth_match_md5), None
         )
         if cached_instance is not None:
             return cached_instance
@@ -349,5 +479,7 @@ class PluginManager:
         plugin: Union[Api, Search, Download, Authentication, Crunch] = plugin_class(
             provider, plugin_conf
         )
-        self._built_plugins_cache[(provider, topic_class.__name__)] = plugin
+        self._built_plugins_cache[
+            (provider, topic_class.__name__, auth_match_md5)
+        ] = plugin
         return plugin

eodag/plugins/search/__init__.py

@@ -36,8 +36,8 @@ class PreparedSearch:
     """An object collecting needed information for search."""
 
     product_type: Optional[str] = None
-    page: int = DEFAULT_PAGE
-    items_per_page: int = DEFAULT_ITEMS_PER_PAGE
+    page: Optional[int] = DEFAULT_PAGE
+    items_per_page: Optional[int] = DEFAULT_ITEMS_PER_PAGE
     auth: Optional[Union[AuthBase, Dict[str, str]]] = None
     auth_plugin: Optional[Authentication] = None
     count: bool = True
@@ -45,10 +45,10 @@ class PreparedSearch:
     info_message: Optional[str] = None
     exception_message: Optional[str] = None
 
-    need_count: bool = field(init=False)
-    query_params: Dict[str, Any] = field(init=False)
-    query_string: str = field(init=False)
-    search_urls: List[str] = field(init=False)
-    product_type_def_params: Dict[str, Any] = field(init=False)
-    total_items_nb: int = field(init=False)
-    sort_by_qs: str = field(init=False)
+    need_count: bool = field(init=False, repr=False)
+    query_params: Dict[str, Any] = field(init=False, repr=False)
+    query_string: str = field(init=False, repr=False)
+    search_urls: List[str] = field(init=False, repr=False)
+    product_type_def_params: Dict[str, Any] = field(init=False, repr=False)
+    total_items_nb: int = field(init=False, repr=False)
+    sort_by_qs: str = field(init=False, repr=False)