eodag 3.0.0b2__py3-none-any.whl → 3.0.1__py3-none-any.whl

eodag/plugins/download/http.py

@@ -19,12 +19,14 @@ from __future__ import annotations
 
 import logging
 import os
+import re
 import shutil
 import tarfile
 import zipfile
 from datetime import datetime
 from email.message import Message
 from itertools import chain
+from json import JSONDecodeError
 from typing import (
     TYPE_CHECKING,
     Any,
@@ -32,7 +34,6 @@ from typing import (
     Iterator,
     List,
     Optional,
-    Tuple,
     TypedDict,
     Union,
     cast,
@@ -44,6 +45,7 @@ import requests
 from lxml import etree
 from requests import RequestException
 from requests.auth import AuthBase
+from requests.structures import CaseInsensitiveDict
 from stream_zip import ZIP_AUTO, stream_zip
 
 from eodag.api.product.metadata_mapping import (
@@ -78,6 +80,7 @@ from eodag.utils.exceptions import (
     DownloadError,
     MisconfiguredError,
     NotAvailableError,
+    RequestError,
     TimeOutError,
 )
 
@@ -97,31 +100,56 @@ class HTTPDownload(Download):
     """HTTPDownload plugin. Handles product download over HTTP protocol
 
     :param provider: provider name
-    :type provider: str
     :param config: Download plugin configuration:
 
-        * ``config.base_uri`` (str) - (optional) default endpoint url
-        * ``config.extract`` (bool) - (optional) extract downloaded archive or not
-        * ``config.auth_error_code`` (int) - (optional) authentication error code
-        * ``config.dl_url_params`` (dict) - (optional) attitional parameters to send in the request
-        * ``config.archive_depth`` (int) - (optional) level in extracted path tree where to find data
-        * ``config.flatten_top_dirs`` (bool) - (optional) flatten directory structure
-        * ``config.ignore_assets`` (bool) - (optional) ignore assets and download using downloadLink
-        * ``config.order_enabled`` (bool) - (optional) wether order is enabled or not if product is `OFFLINE`
-        * ``config.order_method`` (str) - (optional) HTTP request method, GET (default) or POST
-        * ``config.order_headers`` (dict) - (optional) order request headers
-        * ``config.order_on_response`` (dict) - (optional) edit or add new product properties
-        * ``config.order_status`` (:class:`~eodag.config.PluginConfig.OrderStatus`) - (optional) Order status handling
-
-
-    :type config: :class:`~eodag.config.PluginConfig`
+        * :attr:`~eodag.config.PluginConfig.type` (``str``) (**mandatory**): ``HTTPDownload``
+        * :attr:`~eodag.config.PluginConfig.base_uri` (``str``): default endpoint url
+        * :attr:`~eodag.config.PluginConfig.method` (``str``): HTTP request method for the download request (``GET`` or
+          ``POST``); default: ``GET``
+        * :attr:`~eodag.config.PluginConfig.extract` (``bool``): if the content of the downloaded file should be
+          extracted; default: ``True``
+        * :attr:`~eodag.config.PluginConfig.auth_error_code` (``int``): which error code is returned in case of an
+          authentication error
+        * :attr:`~eodag.config.PluginConfig.dl_url_params` (``Dict[str, Any]``): parameters to be
+          added to the query params of the request
+        * :attr:`~eodag.config.PluginConfig.archive_depth` (``int``): level in extracted path tree where to find data;
+          default: ``1``
+        * :attr:`~eodag.config.PluginConfig.flatten_top_dirs` (``bool``): if the directory structure should be
+          flattened; default: ``True``
+        * :attr:`~eodag.config.PluginConfig.ignore_assets` (``bool``): ignore assets and download using downloadLink;
+          default: ``False``
+        * :attr:`~eodag.config.PluginConfig.timeout` (``int``): time to wait until request timeout in seconds;
+          default: ``5``
+        * :attr:`~eodag.config.PluginConfig.ssl_verify` (``bool``): if the ssl certificates should be verified in
+          requests; default: ``True``
+        * :attr:`~eodag.config.PluginConfig.output_extension` (``str``): which extension should be used for the
+          downloaded file
+        * :attr:`~eodag.config.PluginConfig.no_auth_download` (``bool``): if the download should be done without
+          authentication; default: ``True``
+        * :attr:`~eodag.config.PluginConfig.order_enabled` (``bool``): if the product has to be ordered to download it;
+          if this parameter is set to true, a mapping for the orderLink has to be added to the metadata mapping of
+          the search plugin used for the provider; default: ``False``
+        * :attr:`~eodag.config.PluginConfig.order_method` (``str``): HTTP request method for the order request (``GET``
+          or ``POST``); default: ``GET``
+        * :attr:`~eodag.config.PluginConfig.order_headers` (``[Dict[str, str]]``): headers to be added to the order
+          request
+        * :attr:`~eodag.config.PluginConfig.order_on_response` (:class:`~eodag.config.PluginConfig.OrderOnResponse`):
+          a typed dictionary containing the key ``metadata_mapping`` which can be used to add new product properties
+          based on the data in response to the order request
+        * :attr:`~eodag.config.PluginConfig.order_status` (:class:`~eodag.config.PluginConfig.OrderStatus`):
+          configuration to handle the order status; contains information which method to use, how the response data is
+          interpreted, which status corresponds to success, ordered and error and what should be done on success.
+        * :attr:`~eodag.config.PluginConfig.products` (``Dict[str, Dict[str, Any]``): product type specific config; the
+          keys are the product types, the values are dictionaries which can contain the keys
+          :attr:`~eodag.config.PluginConfig.output_extension` and :attr:`~eodag.config.PluginConfig.extract` to
+          overwrite the provider config for a specific product type
 
     """
 
     def __init__(self, provider: str, config: PluginConfig) -> None:
         super(HTTPDownload, self).__init__(provider, config)
 
-    def orderDownload(
+    def order_download(
         self,
         product: EOProduct,
         auth: Optional[AuthBase] = None,
@@ -133,12 +161,13 @@ class HTTPDownload(Download):
         and has `orderLink` in its properties.
         Product ordering can be configured using the following download plugin parameters:
 
-            - **order_enabled**: Wether order is enabled or not (may not use this method
+            - :attr:`~eodag.config.PluginConfig.order_enabled`: Wether order is enabled or not (may not use this method
               if no `orderLink` exists)
 
-            - **order_method**: (optional) HTTP request method, GET (default) or POST
+            - :attr:`~eodag.config.PluginConfig.order_method`: (optional) HTTP request method, GET (default) or POST
 
-            - **order_on_response**: (optional) things to do with obtained order response:
+            - :attr:`~eodag.config.PluginConfig.order_on_response`: (optional) things to do with obtained order
+              response:
 
               - *metadata_mapping*: edit or add new product propoerties properties
 
@@ -147,13 +176,9 @@ class HTTPDownload(Download):
             - **orderLink**: order request URL
 
         :param product: The EO product to order
-        :type product: :class:`~eodag.api.product._product.EOProduct`
         :param auth: (optional) authenticated object
-        :type auth: Optional[AuthBase]
         :param kwargs: download additional kwargs
-        :type kwargs: Union[str, bool, dict]
         :returns: the returned json status response
-        :rtype: dict
         """
         product.properties["storageStatus"] = STAGING_STATUS
 
@@ -167,10 +192,15 @@ class HTTPDownload(Download):
         if order_method == "POST":
             # separate url & parameters
             parts = urlparse(str(product.properties["orderLink"]))
-            query_dict = parse_qs(parts.query)
-            if not query_dict and parts.query:
+            query_dict = {}
+            # `parts.query` may be a JSON with query strings as one of values. If `parse_qs` is executed as first step,
+            # the resulting `query_dict` would be erroneous.
+            try:
                 query_dict = geojson.loads(parts.query)
-            order_url = parts._replace(query=None).geturl()
+            except JSONDecodeError:
+                if parts.query:
+                    query_dict = parse_qs(parts.query)
+            order_url = parts._replace(query="").geturl()
             if query_dict:
                 order_kwargs["json"] = query_dict
         else:
@@ -195,18 +225,11 @@ class HTTPDownload(Download):
                     logger.debug(ordered_message)
                     product.properties["storageStatus"] = STAGING_STATUS
                 except RequestException as e:
-                    if hasattr(e, "response") and (
-                        content := getattr(e.response, "content", None)
-                    ):
-                        error_message = f"{content.decode('utf-8')} - {e}"
-                    else:
-                        error_message = str(e)
-                    logger.warning(
-                        "%s could not be ordered, request returned %s",
-                        product.properties["title"],
-                        error_message,
-                    )
                     self._check_auth_exception(e)
+                    title = product.properties["title"]
+                    message = f"{title} could not be ordered"
+                    raise RequestError.from_error(e, message) from e
+
                 return self.order_response_process(response, product)
         except requests.exceptions.Timeout as exc:
             raise TimeOutError(exc, timeout=timeout) from exc
@@ -217,11 +240,8 @@ class HTTPDownload(Download):
         """Process order response
 
         :param response: The order response
-        :type response: :class:`~requests.Response`
         :param product: The orderd EO product
-        :type product: :class:`~eodag.api.product._product.EOProduct`
         :returns: the returned json status response
-        :rtype: dict
         """
         on_response_mm = getattr(self.config, "order_on_response", {}).get(
             "metadata_mapping", {}
@@ -251,7 +271,7 @@ class HTTPDownload(Download):
 
         return json_response
 
-    def orderDownloadStatus(
+    def order_download_status(
         self,
         product: EOProduct,
         auth: Optional[AuthBase] = None,
@@ -261,18 +281,15 @@ class HTTPDownload(Download):
         It will be executed before each download retry.
         Product order status request can be configured using the following download plugin parameters:
 
-            - **order_status**: :class:`~eodag.config.PluginConfig.OrderStatus`
+            - :attr:`~eodag.config.PluginConfig.order_status`: :class:`~eodag.config.PluginConfig.OrderStatus`
 
         Product properties used for order status:
 
             - **orderStatusLink**: order status request URL
 
         :param product: The ordered EO product
-        :type product: :class:`~eodag.api.product._product.EOProduct`
         :param auth: (optional) authenticated object
-        :type auth: Optional[AuthBase]
         :param kwargs: download additional kwargs
-        :type kwargs: Union[str, bool, dict]
         """
 
         status_config = getattr(self.config, "order_status", {})
@@ -325,7 +342,7 @@ class HTTPDownload(Download):
         if status_request_method == "POST":
             # separate url & parameters
             parts = urlparse(str(product.properties["orderStatusLink"]))
-            status_url = parts._replace(query=None).geturl()
+            status_url = parts._replace(query="").geturl()
             query_dict = parse_qs(parts.query)
             if not query_dict and parts.query:
                 query_dict = geojson.loads(parts.query)
@@ -568,12 +585,12 @@ class HTTPDownload(Download):
             )
             progress_callback = ProgressCallback(disable=True)
 
-        outputs_extension = getattr(self.config, "products", {}).get(
+        output_extension = getattr(self.config, "products", {}).get(
             product.product_type, {}
-        ).get("outputs_extension", None) or getattr(
-            self.config, "outputs_extension", ".zip"
+        ).get("output_extension", None) or getattr(
+            self.config, "output_extension", ".zip"
         )
-        kwargs["outputs_extension"] = kwargs.get("outputs_extension", outputs_extension)
+        kwargs["output_extension"] = kwargs.get("output_extension", output_extension)
 
         fs_path, record_filename = self._prepare_download(
             product,
@@ -637,7 +654,7 @@ class HTTPDownload(Download):
         logger.debug("Download recorded in %s", record_filename)
 
         # Check that the downloaded file is really a zip file
-        if not zipfile.is_zipfile(fs_path) and outputs_extension == ".zip":
+        if not zipfile.is_zipfile(fs_path) and output_extension == ".zip":
             logger.warning(
                 "Downloaded product is not a Zip File. Please check its file type before using it"
             )
@@ -658,7 +675,7 @@ class HTTPDownload(Download):
                 if not new_fs_path.endswith(".tar"):
                     new_fs_path += ".tar"
                 shutil.move(fs_path, new_fs_path)
-                kwargs["outputs_extension"] = ".tar"
+                kwargs["output_extension"] = ".tar"
                 product_path = self._finalize(
                     new_fs_path,
                     progress_callback=progress_callback,
@@ -727,13 +744,13 @@ class HTTPDownload(Download):
                     if ext:
                         filename += ext
                 else:
-                    outputs_extension: Optional[str] = (
+                    output_extension: Optional[str] = (
                         getattr(self.config, "products", {})
                         .get(product.product_type, {})
-                        .get("outputs_extension")
+                        .get("output_extension")
                     )
-                    if outputs_extension:
-                        filename += outputs_extension
+                    if output_extension:
+                        filename += output_extension
 
         return filename
 
@@ -747,27 +764,20 @@ class HTTPDownload(Download):
         **kwargs: Unpack[DownloadConf],
     ) -> StreamResponse:
         r"""
-        Returns dictionnary of :class:`~fastapi.responses.StreamingResponse` keyword-arguments.
+        Returns dictionary of :class:`~fastapi.responses.StreamingResponse` keyword-arguments.
         It contains a generator to streamed download chunks and the response headers.
 
         :param product: The EO product to download
-        :type product: :class:`~eodag.api.product._product.EOProduct`
         :param auth: (optional) authenticated object
-        :type auth: Optional[Union[AuthBase, Dict[str, str]]]
         :param progress_callback: (optional) A progress callback
-        :type progress_callback: :class:`~eodag.utils.ProgressCallback`
         :param wait: (optional) If download fails, wait time in minutes between two download tries
-        :type wait: int
         :param timeout: (optional) If download fails, maximum time in minutes before stop retrying
                         to download
-        :type timeout: int
-        :param kwargs: `outputs_prefix` (str), `extract` (bool), `delete_archive` (bool)
+        :param kwargs: `output_dir` (str), `extract` (bool), `delete_archive` (bool)
                         and `dl_url_params` (dict) can be provided as additional kwargs
                         and will override any other values defined in a configuration
                         file or with environment variables.
-        :type kwargs: Union[str, bool, dict]
-        :returns: Dictionnary of :class:`~fastapi.responses.StreamingResponse` keyword-arguments
-        :rtype: dict
+        :returns: Dictionary of :class:`~fastapi.responses.StreamingResponse` keyword-arguments
         """
         if auth is not None and not isinstance(auth, AuthBase):
             raise MisconfiguredError(f"Incompatible auth plugin: {type(auth)}")
@@ -852,12 +862,9 @@ class HTTPDownload(Download):
             and e.response.status_code in auth_errors
         ):
             raise AuthenticationError(
-                "HTTP Error %s returned, %s\nPlease check your credentials for %s"
-                % (
-                    e.response.status_code,
-                    response_text,
-                    self.provider,
-                )
+                f"Please check your credentials for {self.provider}.",
+                f"HTTP Error {e.response.status_code} returned.",
+                response_text,
             )
 
     def _process_exception(
@@ -906,35 +913,33 @@ class HTTPDownload(Download):
         fetches a zip file containing the assets of a given product as a stream
         and returns a generator yielding the chunks of the file
         :param product: product for which the assets should be downloaded
-        :type product: :class:`~eodag.api.product._product.EOProduct`
         :param auth: The configuration of a plugin of type Authentication
-        :type auth: Optional[Union[AuthBase, Dict[str, str]]]
         :param progress_callback: A method or a callable object
                                   which takes a current size and a maximum
                                   size as inputs and handle progress bar
                                   creation and update to give the user a
                                   feedback on the download progress
-        :type progress_callback: :class:`~eodag.utils.ProgressCallback`
         :param kwargs: additional arguments
-        :type kwargs: dict
         """
         if progress_callback is None:
             logger.info("Progress bar unavailable, please call product.download()")
             progress_callback = ProgressCallback(disable=True)
 
+        ssl_verify = getattr(self.config, "ssl_verify", True)
+
         ordered_message = ""
         if (
             "orderLink" in product.properties
             and product.properties.get("storageStatus") == OFFLINE_STATUS
             and not product.properties.get("orderStatus")
         ):
-            self.orderDownload(product=product, auth=auth)
+            self.order_download(product=product, auth=auth)
 
         if (
             product.properties.get("orderStatusLink", None)
             and product.properties.get("storageStatus") != ONLINE_STATUS
         ):
-            self.orderDownloadStatus(product=product, auth=auth)
+            self.order_download_status(product=product, auth=auth)
 
         params = kwargs.pop("dl_url_params", None) or getattr(
             self.config, "dl_url_params", {}
@@ -951,7 +956,7 @@ class HTTPDownload(Download):
             query_dict = parse_qs(parts.query)
             if not query_dict and parts.query:
                 query_dict = geojson.loads(parts.query)
-            req_url = parts._replace(query=None).geturl()
+            req_url = parts._replace(query="").geturl()
             req_kwargs: Dict[str, Any] = {"json": query_dict} if query_dict else {}
         else:
             req_url = url
@@ -972,6 +977,7 @@ class HTTPDownload(Download):
             params=params,
             headers=USER_AGENT,
             timeout=DEFAULT_STREAM_REQUESTS_TIMEOUT,
+            verify=ssl_verify,
             **req_kwargs,
         ) as self.stream:
             try:
@@ -1004,8 +1010,11 @@ class HTTPDownload(Download):
                     "content-disposition"
                 ] = f"attachment; filename={filename}"
                 content_type = product.headers.get("Content-Type")
-                if filename and not content_type:
-                    product.headers["Content-Type"] = guess_file_type(filename)
+                guessed_content_type = (
+                    guess_file_type(filename) if filename and not content_type else None
+                )
+                if guessed_content_type is not None:
+                    product.headers["Content-Type"] = guessed_content_type
 
                 progress_callback.reset(total=stream_size)
                 for chunk in self.stream.iter_content(chunk_size=64 * 1024):
@@ -1020,7 +1029,7 @@ class HTTPDownload(Download):
         progress_callback: Optional[ProgressCallback] = None,
         assets_values: List[Asset] = [],
         **kwargs: Unpack[DownloadConf],
-    ) -> Iterator[Tuple[str, datetime, int, Any, Iterator[Any]]]:
+    ) -> Iterator[Any]:
         if progress_callback is None:
             logger.info("Progress bar unavailable, please call product.download()")
             progress_callback = ProgressCallback(disable=True)
@@ -1071,20 +1080,34 @@ class HTTPDownload(Download):
             "flatten_top_dirs", getattr(self.config, "flatten_top_dirs", True)
         )
         ssl_verify = getattr(self.config, "ssl_verify", True)
+        matching_url = (
+            getattr(product.downloader_auth.config, "matching_url", "")
+            if product.downloader_auth
+            else ""
+        )
+        matching_conf = (
+            getattr(product.downloader_auth.config, "matching_conf", None)
+            if product.downloader_auth
+            else None
+        )
 
         # loop for assets download
         for asset in assets_values:
-
             if not asset["href"] or asset["href"].startswith("file:"):
                 logger.info(
                     f"Local asset detected. Download skipped for {asset['href']}"
                 )
                 continue
-
+            if matching_conf or (
+                matching_url and re.match(matching_url, asset["href"])
+            ):
+                auth_object = auth
+            else:
+                auth_object = None
             with requests.get(
                 asset["href"],
                 stream=True,
-                auth=auth,
+                auth=auth_object,
                 params=params,
                 headers=USER_AGENT,
                 timeout=DEFAULT_STREAM_REQUESTS_TIMEOUT,
@@ -1097,8 +1120,7 @@ class HTTPDownload(Download):
                         exc, timeout=DEFAULT_STREAM_REQUESTS_TIMEOUT
                     ) from exc
                 except RequestException as e:
-                    raise_errors = True if len(assets_values) == 1 else False
-                    self._handle_asset_exception(e, asset, raise_errors=raise_errors)
+                    self._handle_asset_exception(e, asset)
                 else:
                     asset_rel_path = (
                         asset.rel_path.replace(assets_common_subdir, "").strip(os.sep)
@@ -1194,7 +1216,9 @@ class HTTPDownload(Download):
             # start reading chunks to set asset.rel_path
             first_chunks_tuple = next(chunks_tuples)
             chunks = chain(iter([first_chunks_tuple]), chunks_tuples)
-            chunks_tuples = [(assets_values[0].rel_path, None, None, None, chunks)]
+            chunks_tuples = iter(
+                [(assets_values[0].rel_path, None, None, None, chunks)]
+            )
 
         for chunk_tuple in chunks_tuples:
             asset_path = chunk_tuple[0]
@@ -1254,27 +1278,22 @@ class HTTPDownload(Download):
 
         return fs_dir_path
 
-    def _handle_asset_exception(
-        self, e: RequestException, asset: Asset, raise_errors: bool = False
-    ) -> None:
+    def _handle_asset_exception(self, e: RequestException, asset: Asset) -> None:
         # check if error is identified as auth_error in provider conf
         auth_errors = getattr(self.config, "auth_error_code", [None])
         if not isinstance(auth_errors, list):
             auth_errors = [auth_errors]
         if e.response is not None and e.response.status_code in auth_errors:
             raise AuthenticationError(
-                "HTTP Error %s returned, %s\nPlease check your credentials for %s"
-                % (
-                    e.response.status_code,
-                    e.response.text.strip(),
-                    self.provider,
-                )
+                f"Please check your credentials for {self.provider}.",
+                f"HTTP Error {e.response.status_code} returned.",
+                e.response.text.strip(),
             )
-        elif raise_errors:
-            raise DownloadError(e)
         else:
-            logger.warning("Unexpected error: %s" % e)
-            logger.warning("Skipping %s" % asset["href"])
+            logger.error(
+                "Unexpected error at download of asset %s: %s", asset["href"], e
+            )
+            raise DownloadError(e)
 
     def _get_asset_sizes(
         self,
@@ -1291,12 +1310,18 @@ class HTTPDownload(Download):
         for asset in assets_values:
             if asset["href"] and not asset["href"].startswith("file:"):
                 # HEAD request for size & filename
-                asset_headers = requests.head(
-                    asset["href"],
-                    auth=auth,
-                    headers=USER_AGENT,
-                    timeout=timeout,
-                ).headers
+                try:
+                    asset_headers = requests.head(
+                        asset["href"],
+                        auth=auth,
+                        params=params,
+                        headers=USER_AGENT,
+                        timeout=timeout,
+                        verify=ssl_verify,
+                    ).headers
+                except RequestException as e:
+                    logger.debug(f"HEAD request failed: {str(e)}")
+                    asset_headers = CaseInsensitiveDict()
 
                 if not getattr(asset, "size", 0):
                     # size from HEAD header / Content-length