eodag 2.12.1__py3-none-any.whl → 3.0.0b2__py3-none-any.whl

eodag/plugins/download/s3rest.py

@@ -17,16 +17,16 @@
 # limitations under the License.
 from __future__ import annotations
 
-import hashlib
 import logging
 import os
 import os.path
-from typing import TYPE_CHECKING, Any, Dict, Optional, Union
+from typing import TYPE_CHECKING, Dict, Optional, Union
 from xml.dom import minidom
 from xml.parsers.expat import ExpatError
 
 import requests
 from requests import RequestException
+from requests.auth import AuthBase
 
 from eodag.api.product.metadata_mapping import OFFLINE_STATUS, ONLINE_STATUS
 from eodag.plugins.download.base import Download
@@ -46,6 +46,7 @@ from eodag.utils import (
 from eodag.utils.exceptions import (
     AuthenticationError,
     DownloadError,
+    MisconfiguredError,
     NotAvailableError,
     RequestError,
 )
@@ -53,6 +54,8 @@ from eodag.utils.exceptions import (
 if TYPE_CHECKING:
     from eodag.api.product import EOProduct
     from eodag.config import PluginConfig
+    from eodag.types.download_args import DownloadConf
+    from eodag.utils import Unpack
 
 logger = logging.getLogger("eodag.download.s3rest")
 
@@ -76,10 +79,7 @@ class S3RestDownload(Download):
         * ``config.order_method`` (str) - (optional) HTTP request method, GET (default) or POST
         * ``config.order_headers`` (dict) - (optional) order request headers
         * ``config.order_on_response`` (dict) - (optional) edit or add new product properties
-        * ``config.order_status_method`` (str) - (optional) status HTTP request method, GET (default) or POST
-        * ``config.order_status_percent`` (str) - (optional) progress percentage key in obtained status response
-        * ``config.order_status_success`` (dict) - (optional) key/value identifying an error success
-        * ``config.order_status_on_success`` (dict) - (optional) edit or add new product properties
+        * ``config.order_status`` (:class:`~eodag.config.PluginConfig.OrderStatus`) - Order status handling
 
     :type config: :class:`~eodag.config.PluginConfig`
     """
@@ -91,18 +91,18 @@ class S3RestDownload(Download):
     def download(
         self,
         product: EOProduct,
-        auth: Optional[PluginConfig] = None,
+        auth: Optional[Union[AuthBase, Dict[str, str]]] = None,
         progress_callback: Optional[ProgressCallback] = None,
         wait: int = DEFAULT_DOWNLOAD_WAIT,
         timeout: int = DEFAULT_DOWNLOAD_TIMEOUT,
-        **kwargs: Union[str, bool, Dict[str, Any]],
+        **kwargs: Unpack[DownloadConf],
     ) -> Optional[str]:
         """Download method for S3 REST API.
 
         :param product: The EO product to download
         :type product: :class:`~eodag.api.product._product.EOProduct`
-        :param auth: (optional) The configuration of a plugin of type Authentication
-        :type auth: :class:`~eodag.config.PluginConfig`
+        :param auth: (optional) authenticated object
+        :type auth: Optional[Union[AuthBase, Dict[str, str]]]
         :param progress_callback: (optional) A method or a callable object
                                   which takes a current size and a maximum
                                   size as inputs and handle progress bar
@@ -117,6 +117,9 @@ class S3RestDownload(Download):
         :returns: The absolute path to the downloaded product in the local filesystem
         :rtype: str
         """
+        if auth is not None and not isinstance(auth, AuthBase):
+            raise MisconfiguredError(f"Incompatible auth plugin: {type(auth)}")
+
         if progress_callback is None:
             logger.info(
                 "Progress bar unavailable, please call product.download() instead of plugin.download()"
@@ -135,10 +138,10 @@ class S3RestDownload(Download):
         @self._download_retry(product, wait, timeout)
         def download_request(
             product: EOProduct,
-            auth: PluginConfig,
+            auth: AuthBase,
             progress_callback: ProgressCallback,
             ordered_message: str,
-            **kwargs: Any,
+            **kwargs: Unpack[DownloadConf],
         ):
             # check order status
             if product.properties.get("orderStatusLink", None):
@@ -172,8 +175,16 @@ class S3RestDownload(Download):
 
             # get nodes/files list contained in the bucket
             logger.debug("Retrieving product content from %s", nodes_list_url)
+
+            ssl_verify = getattr(self.config, "ssl_verify", True)
+            timeout = getattr(self.config, "timeout", HTTP_REQ_TIMEOUT)
+
             bucket_contents = requests.get(
-                nodes_list_url, auth=auth, headers=USER_AGENT, timeout=HTTP_REQ_TIMEOUT
+                nodes_list_url,
+                auth=auth,
+                headers=USER_AGENT,
+                timeout=timeout,
+                verify=ssl_verify,
             )
             try:
                 bucket_contents.raise_for_status()
@@ -252,8 +263,9 @@ class S3RestDownload(Download):
                         "Unable to create records directory. Got:\n%s", tb.format_exc()
                     )
             # check if product has already been downloaded
-            url_hash = hashlib.md5(product.remote_location.encode("utf-8")).hexdigest()
-            record_filename = os.path.join(download_records_dir, url_hash)
+            record_filename = os.path.join(
+                download_records_dir, self.generate_record_hash(product)
+            )
             if os.path.isfile(record_filename) and os.path.exists(product_local_path):
                 product.location = path_to_uri(product_local_path)
                 return product_local_path
@@ -303,6 +315,7 @@ class S3RestDownload(Download):
                     auth=auth,
                     headers=USER_AGENT,
                     timeout=DEFAULT_STREAM_REQUESTS_TIMEOUT,
+                    verify=ssl_verify,
                 ) as stream:
                     try:
                         stream.raise_for_status()

eodag/plugins/manager.py

@@ -43,7 +43,7 @@ from eodag.plugins.crunch.base import Crunch
 from eodag.plugins.download.base import Download
 from eodag.plugins.search.base import Search
 from eodag.utils import GENERIC_PRODUCT_TYPE
-from eodag.utils.exceptions import UnsupportedProvider
+from eodag.utils.exceptions import MisconfiguredError, UnsupportedProvider
 
 if TYPE_CHECKING:
     from eodag.api.product import EOProduct
@@ -73,7 +73,12 @@ class PluginManager:
 
     supported_topics = {"search", "download", "crunch", "auth", "api"}
 
+    product_type_to_provider_config_map: Dict[str, List[ProviderConfig]]
+
+    skipped_plugins: List[str]
+
     def __init__(self, providers_config: Dict[str, ProviderConfig]) -> None:
+        self.skipped_plugins = []
         self.providers_config = providers_config
         # Load all the plugins. This will make all plugin classes of a particular
         # type to be available in the base plugin class's 'plugins' attribute.
@@ -90,6 +95,13 @@ class PluginManager:
             ):
                 try:
                     entry_point.load()
+                except pkg_resources.DistributionNotFound:
+                    logger.debug(
+                        "%s plugin skipped, eodag[%s] or eodag[all] needed",
+                        entry_point.name,
+                        ",".join(entry_point.extras),
+                    )
+                    self.skipped_plugins.append(entry_point.name)
                 except ImportError:
                     import traceback as tb
 
@@ -116,7 +128,9 @@ class PluginManager:
                         self.providers_config = plugin_providers_config
         self.rebuild()
 
-    def rebuild(self, providers_config=None):
+    def rebuild(
+        self, providers_config: Optional[Dict[str, ProviderConfig]] = None
+    ) -> None:
         """(Re)Build plugin manager mapping and cache"""
         if providers_config is not None:
             self.providers_config = providers_config
@@ -126,13 +140,13 @@ class PluginManager:
 
     def build_product_type_to_provider_config_map(self) -> None:
         """Build mapping conf between product types and providers"""
-        self.product_type_to_provider_config_map: Dict[str, List[ProviderConfig]] = {}
+        self.product_type_to_provider_config_map = {}
         for provider in list(self.providers_config):
             provider_config = self.providers_config[provider]
             if not hasattr(provider_config, "products") or not provider_config.products:
                 logger.info(
-                    "%s: provider has no product configured and will be skipped"
-                    % provider
+                    "%s: provider has no product configured and will be skipped",
+                    provider,
                 )
                 self.providers_config.pop(provider)
                 continue
@@ -159,56 +173,57 @@ class PluginManager:
         :param product_type: (optional) The product type that the constructed plugins
                              must support
         :type product_type: str
-        :param provider: (optional) The provider on which to get the search plugin
+        :param provider: (optional) The provider or the provider group on which to get
+            the search plugins
         :type provider: str
         :returns: All the plugins supporting the product type, one by one (a generator
                   object)
-        :rtype: types.GeneratorType(:class:`~eodag.plugins.search.Search` or :class:`~eodag.plugins.download.Api`)
+        :rtype: types.GeneratorType(:class:`~eodag.plugins.search.Search`
+            or :class:`~eodag.plugins.download.Api`)
         :raises: :class:`~eodag.utils.exceptions.UnsupportedProvider`
-        :raises: :class:`~eodag.utils.exceptions.UnsupportedProductType`
         """
 
         def get_plugin() -> Union[Search, Api]:
             plugin: Union[Search, Api]
-            try:
+            if search := getattr(config, "search", None):
                 config.search.products = config.products
                 config.search.priority = config.priority
-                plugin = cast(
-                    Search, self._build_plugin(config.name, config.search, Search)
-                )
-            except AttributeError:
+                plugin = cast(Search, self._build_plugin(config.name, search, Search))
+            elif api := getattr(config, "api", None):
                 config.api.products = config.products
                 config.api.priority = config.priority
-                plugin = cast(Api, self._build_plugin(config.name, config.api, Api))
+                plugin = cast(Api, self._build_plugin(config.name, api, Api))
+            else:
+                raise MisconfiguredError(
+                    f"No search plugin configureed for {config.name}."
+                )
             return plugin
 
-        if provider is not None:
-            try:
-                config = self.providers_config[provider]
-            except KeyError:
-                raise UnsupportedProvider
-            yield get_plugin()
-            # Signal the end of iteration as we already have what we wanted (see PEP-479)
-            return
-
-        if product_type is None:
-            for config in sorted(
-                self.providers_config.values(), key=attrgetter("priority"), reverse=True
-            ):
-                yield get_plugin()
-            # Signal the end of iteration as we already have what we wanted (see PEP-479)
-            return
-        try:
-            for config in self.product_type_to_provider_config_map[product_type]:
-                yield get_plugin()
-        except KeyError:
-            logger.info(
-                "UnsupportedProductType: %s, using generic settings", product_type
+        configs: Optional[List[ProviderConfig]]
+        if product_type:
+            configs = self.product_type_to_provider_config_map.get(product_type)
+            if not configs:
+                logger.info(
+                    "UnsupportedProductType: %s, using generic settings", product_type
+                )
+                configs = self.product_type_to_provider_config_map[GENERIC_PRODUCT_TYPE]
+        else:
+            configs = list(self.providers_config.values())
+
+        if provider:
+            configs = [
+                c for c in configs if provider in [getattr(c, "group", None), c.name]
+            ]
+
+        if not configs and product_type:
+            raise UnsupportedProvider(
+                f"{provider} is not (yet) supported for {product_type}"
             )
-            for config in self.product_type_to_provider_config_map[
-                GENERIC_PRODUCT_TYPE
-            ]:
-                yield get_plugin()
+        if not configs:
+            raise UnsupportedProvider(f"{provider} is not (yet) supported")
+
+        for config in sorted(configs, key=attrgetter("priority"), reverse=True):
+            yield get_plugin()
 
     def get_download_plugin(self, product: EOProduct) -> Union[Download, Api]:
         """Build and return the download plugin capable of downloading the given
@@ -220,19 +235,21 @@ class PluginManager:
         :rtype: :class:`~eodag.plugins.download.Download` or :class:`~eodag.plugins.download.Api`
         """
         plugin_conf = self.providers_config[product.provider]
-        try:
+        if download := getattr(plugin_conf, "download", None):
             plugin_conf.download.priority = plugin_conf.priority
             plugin = cast(
                 Download,
-                self._build_plugin(product.provider, plugin_conf.download, Download),
+                self._build_plugin(product.provider, download, Download),
             )
-            return plugin
-        except AttributeError:
+        elif api := getattr(plugin_conf, "api", None):
+            plugin_conf.api.products = plugin_conf.products
             plugin_conf.api.priority = plugin_conf.priority
-            plugin = cast(
-                Api, self._build_plugin(product.provider, plugin_conf.api, Api)
+            plugin = cast(Api, self._build_plugin(product.provider, api, Api))
+        else:
+            raise MisconfiguredError(
+                f"No download plugin configured for provider {plugin_conf.name}."
             )
-            return plugin
+        return plugin
 
     def get_auth_plugin(self, provider: str) -> Optional[Authentication]:
         """Build and return the authentication plugin for the given product_type and
@@ -244,17 +261,17 @@ class PluginManager:
         :rtype: :class:`~eodag.plugins.authentication.Authentication`
         """
         plugin_conf = self.providers_config[provider]
-        try:
-            plugin_conf.auth.priority = plugin_conf.priority
-            plugin = cast(
-                Authentication,
-                self._build_plugin(provider, plugin_conf.auth, Authentication),
-            )
-            return plugin
-        except AttributeError:
+        auth: Optional[PluginConfig] = getattr(plugin_conf, "auth", None)
+        if not auth:
             # We guess the plugin being built is of type Api, therefore no need
             # for an Auth plugin.
             return None
+        auth.priority = plugin_conf.priority
+        plugin = cast(
+            Authentication,
+            self._build_plugin(provider, auth, Authentication),
+        )
+        return plugin
 
     @staticmethod
     def get_crunch_plugin(name: str, **options: Any) -> Crunch:
@@ -268,8 +285,8 @@ class PluginManager:
         :returns: The cruncher named `name`
         :rtype: :class:`~eodag.plugins.crunch.Crunch`
         """
-        Klass = Crunch.get_plugin_by_class_name(name)
-        return Klass(options)
+        klass = Crunch.get_plugin_by_class_name(name)
+        return klass(options)
 
     def sort_providers(self) -> None:
         """Sort providers taking into account current priority order"""

eodag/plugins/search/__init__.py

@@ -16,3 +16,39 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 """EODAG search package"""
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+from typing import TYPE_CHECKING
+
+from eodag.utils import DEFAULT_ITEMS_PER_PAGE, DEFAULT_PAGE
+
+if TYPE_CHECKING:
+    from typing import Any, Dict, List, Optional, Union
+
+    from requests.auth import AuthBase
+
+    from eodag.plugins.authentication.base import Authentication
+
+
+@dataclass
+class PreparedSearch:
+    """An object collecting needed information for search."""
+
+    product_type: Optional[str] = None
+    page: int = DEFAULT_PAGE
+    items_per_page: int = DEFAULT_ITEMS_PER_PAGE
+    auth: Optional[Union[AuthBase, Dict[str, str]]] = None
+    auth_plugin: Optional[Authentication] = None
+    count: bool = True
+    url: Optional[str] = None
+    info_message: Optional[str] = None
+    exception_message: Optional[str] = None
+
+    need_count: bool = field(init=False)
+    query_params: Dict[str, Any] = field(init=False)
+    query_string: str = field(init=False)
+    search_urls: List[str] = field(init=False)
+    product_type_def_params: Dict[str, Any] = field(init=False)
+    total_items_nb: int = field(init=False)
+    sort_by_qs: str = field(init=False)