eodag 2.12.0__py3-none-any.whl → 3.0.0__py3-none-any.whl

eodag/plugins/download/s3rest.py

@@ -17,16 +17,16 @@
 # limitations under the License.
 from __future__ import annotations
 
-import hashlib
 import logging
 import os
 import os.path
-from typing import TYPE_CHECKING, Any, Dict, Optional, Union
+from typing import TYPE_CHECKING, Dict, List, Optional, Union
 from xml.dom import minidom
 from xml.parsers.expat import ExpatError
 
 import requests
 from requests import RequestException
+from requests.auth import AuthBase
 
 from eodag.api.product.metadata_mapping import OFFLINE_STATUS, ONLINE_STATUS
 from eodag.plugins.download.base import Download
@@ -46,6 +46,7 @@ from eodag.utils import (
 from eodag.utils.exceptions import (
     AuthenticationError,
     DownloadError,
+    MisconfiguredError,
     NotAvailableError,
     RequestError,
 )
@@ -53,6 +54,8 @@ from eodag.utils.exceptions import (
 if TYPE_CHECKING:
     from eodag.api.product import EOProduct
     from eodag.config import PluginConfig
+    from eodag.types.download_args import DownloadConf
+    from eodag.utils import Unpack
 
 logger = logging.getLogger("eodag.download.s3rest")
 
@@ -65,7 +68,6 @@ class S3RestDownload(Download):
     Re-use AwsDownload bucket some handling methods
 
     :param provider: provider name
-    :type provider: str
     :param config: Download plugin configuration:
 
         * ``config.base_uri`` (str) - default endpoint url
@@ -76,12 +78,7 @@ class S3RestDownload(Download):
         * ``config.order_method`` (str) - (optional) HTTP request method, GET (default) or POST
         * ``config.order_headers`` (dict) - (optional) order request headers
         * ``config.order_on_response`` (dict) - (optional) edit or add new product properties
-        * ``config.order_status_method`` (str) - (optional) status HTTP request method, GET (default) or POST
-        * ``config.order_status_percent`` (str) - (optional) progress percentage key in obtained status response
-        * ``config.order_status_success`` (dict) - (optional) key/value identifying an error success
-        * ``config.order_status_on_success`` (dict) - (optional) edit or add new product properties
-
-    :type config: :class:`~eodag.config.PluginConfig`
+        * ``config.order_status`` (:class:`~eodag.config.PluginConfig.OrderStatus`) - Order status handling
     """
 
     def __init__(self, provider: str, config: PluginConfig) -> None:
@@ -91,32 +88,30 @@ class S3RestDownload(Download):
     def download(
         self,
         product: EOProduct,
-        auth: Optional[PluginConfig] = None,
+        auth: Optional[Union[AuthBase, Dict[str, str]]] = None,
         progress_callback: Optional[ProgressCallback] = None,
         wait: int = DEFAULT_DOWNLOAD_WAIT,
         timeout: int = DEFAULT_DOWNLOAD_TIMEOUT,
-        **kwargs: Union[str, bool, Dict[str, Any]],
+        **kwargs: Unpack[DownloadConf],
     ) -> Optional[str]:
         """Download method for S3 REST API.
 
         :param product: The EO product to download
-        :type product: :class:`~eodag.api.product._product.EOProduct`
-        :param auth: (optional) The configuration of a plugin of type Authentication
-        :type auth: :class:`~eodag.config.PluginConfig`
+        :param auth: (optional) authenticated object
         :param progress_callback: (optional) A method or a callable object
                                   which takes a current size and a maximum
                                   size as inputs and handle progress bar
                                   creation and update to give the user a
                                   feedback on the download progress
-        :type progress_callback: :class:`~eodag.utils.ProgressCallback` or None
-        :param kwargs: `outputs_prefix` (str), `extract` (bool), `delete_archive` (bool)
+        :param kwargs: `output_dir` (str), `extract` (bool), `delete_archive` (bool)
                         and `dl_url_params` (dict) can be provided as additional kwargs
                         and will override any other values defined in a configuration
                         file or with environment variables.
-        :type kwargs: Union[str, bool, dict]
         :returns: The absolute path to the downloaded product in the local filesystem
-        :rtype: str
         """
+        if auth is not None and not isinstance(auth, AuthBase):
+            raise MisconfiguredError(f"Incompatible auth plugin: {type(auth)}")
+
         if progress_callback is None:
             logger.info(
                 "Progress bar unavailable, please call product.download() instead of plugin.download()"
@@ -130,19 +125,19 @@ class S3RestDownload(Download):
             and "storageStatus" in product.properties
             and product.properties["storageStatus"] != ONLINE_STATUS
         ):
-            self.http_download_plugin.orderDownload(product=product, auth=auth)
+            self.http_download_plugin.order_download(product=product, auth=auth)
 
         @self._download_retry(product, wait, timeout)
         def download_request(
             product: EOProduct,
-            auth: PluginConfig,
+            auth: AuthBase,
             progress_callback: ProgressCallback,
             ordered_message: str,
-            **kwargs: Any,
+            **kwargs: Unpack[DownloadConf],
         ):
             # check order status
             if product.properties.get("orderStatusLink", None):
-                self.http_download_plugin.orderDownloadStatus(
+                self.http_download_plugin.order_download_status(
                     product=product, auth=auth
                 )
 
@@ -150,6 +145,8 @@ class S3RestDownload(Download):
             bucket_name, prefix = get_bucket_name_and_prefix(
                 url=product.location, bucket_path_level=self.config.bucket_path_level
             )
+            if prefix is None:
+                raise DownloadError(f"Could not extract prefix from {product.location}")
 
             if (
                 bucket_name is None
@@ -172,8 +169,16 @@ class S3RestDownload(Download):
 
             # get nodes/files list contained in the bucket
             logger.debug("Retrieving product content from %s", nodes_list_url)
+
+            ssl_verify = getattr(self.config, "ssl_verify", True)
+            timeout = getattr(self.config, "timeout", HTTP_REQ_TIMEOUT)
+
             bucket_contents = requests.get(
-                nodes_list_url, auth=auth, headers=USER_AGENT, timeout=HTTP_REQ_TIMEOUT
+                nodes_list_url,
+                auth=auth,
+                headers=USER_AGENT,
+                timeout=timeout,
+                verify=ssl_verify,
             )
             try:
                 bucket_contents.raise_for_status()
@@ -184,12 +189,9 @@ class S3RestDownload(Download):
                     auth_errors = [auth_errors]
                 if err.response and err.response.status_code in auth_errors:
                     raise AuthenticationError(
-                        "HTTP Error %s returned, %s\nPlease check your credentials for %s"
-                        % (
-                            err.response.status_code,
-                            err.response.text.strip(),
-                            self.provider,
-                        )
+                        f"Please check your credentials for {self.provider}.",
+                        f"HTTP Error {err.response.status_code} returned.",
+                        err.response.text.strip(),
                     )
                 # product not available
                 elif (
@@ -220,7 +222,7 @@ class S3RestDownload(Download):
                         self.__class__.__name__,
                         bucket_contents.text,
                     )
-                    raise RequestError(str(err))
+                    raise RequestError.from_error(err) from err
             try:
                 xmldoc = minidom.parseString(bucket_contents.text)
             except ExpatError as err:
@@ -232,14 +234,12 @@ class S3RestDownload(Download):
                 logger.warning("Could not load any content from %s", nodes_list_url)
 
             # destination product path
-            outputs_prefix = (
-                kwargs.pop("outputs_prefix", None) or self.config.outputs_prefix
-            )
-            abs_outputs_prefix = os.path.abspath(outputs_prefix)
-            product_local_path = os.path.join(abs_outputs_prefix, prefix.split("/")[-1])
+            output_dir = kwargs.pop("output_dir", None) or self.config.output_dir
+            abs_output_dir = os.path.abspath(output_dir)
+            product_local_path = os.path.join(abs_output_dir, prefix.split("/")[-1])
 
             # .downloaded cache record directory
-            download_records_dir = os.path.join(abs_outputs_prefix, ".downloaded")
+            download_records_dir = os.path.join(abs_output_dir, ".downloaded")
             try:
                 os.makedirs(download_records_dir)
             except OSError as exc:
@@ -252,8 +252,9 @@ class S3RestDownload(Download):
                         "Unable to create records directory. Got:\n%s", tb.format_exc()
                     )
             # check if product has already been downloaded
-            url_hash = hashlib.md5(product.remote_location.encode("utf-8")).hexdigest()
-            record_filename = os.path.join(download_records_dir, url_hash)
+            record_filename = os.path.join(
+                download_records_dir, self.generate_record_hash(product)
+            )
             if os.path.isfile(record_filename) and os.path.exists(product_local_path):
                 product.location = path_to_uri(product_local_path)
                 return product_local_path
@@ -266,18 +267,18 @@ class S3RestDownload(Download):
                 os.remove(record_filename)
 
             # total size for progress_callback
-            total_size = sum(
-                [
-                    int(node.firstChild.nodeValue)
-                    for node in xmldoc.getElementsByTagName("Size")
-                ]
-            )
+            size_list: List[int] = [
+                int(node.firstChild.nodeValue)  # type: ignore[attr-defined]
+                for node in xmldoc.getElementsByTagName("Size")
+                if node.firstChild is not None
+            ]
+            total_size = sum(size_list)
             progress_callback.reset(total=total_size)
 
             # download each node key
             for node_xml in nodes_xml_list:
                 node_key = unquote(
-                    node_xml.getElementsByTagName("Key")[0].firstChild.nodeValue
+                    node_xml.getElementsByTagName("Key")[0].firstChild.nodeValue  # type: ignore[union-attr]
                 )
                 # As "Key", "Size" and "ETag" (md5 hash) can also be retrieved from node_xml
                 node_url = urljoin(bucket_url.strip("/") + "/", node_key.strip("/"))
@@ -303,6 +304,7 @@ class S3RestDownload(Download):
                     auth=auth,
                     headers=USER_AGENT,
                     timeout=DEFAULT_STREAM_REQUESTS_TIMEOUT,
+                    verify=ssl_verify,
                 ) as stream:
                     try:
                         stream.raise_for_status()