eodag 3.0.0b3__py3-none-any.whl → 3.1.0__py3-none-any.whl

eodag/plugins/download/http.py

@@ -19,6 +19,7 @@ from __future__ import annotations
 
 import logging
 import os
+import re
 import shutil
 import tarfile
 import zipfile
@@ -26,17 +27,8 @@ from datetime import datetime
 from email.message import Message
 from itertools import chain
 from json import JSONDecodeError
-from typing import (
-    TYPE_CHECKING,
-    Any,
-    Dict,
-    Iterator,
-    List,
-    Optional,
-    TypedDict,
-    Union,
-    cast,
-)
+from pathlib import Path
+from typing import TYPE_CHECKING, Any, Iterator, Optional, TypedDict, Union, cast
 from urllib.parse import parse_qs, urlparse
 
 import geojson
@@ -70,6 +62,7 @@ from eodag.utils import (
     guess_file_type,
     parse_header,
     path_to_uri,
+    rename_with_version,
     sanitize,
     string_to_jsonpath,
     uri_to_path,
@@ -80,6 +73,7 @@ from eodag.utils.exceptions import (
     MisconfiguredError,
     NotAvailableError,
     TimeOutError,
+    ValidationError,
 )
 
 if TYPE_CHECKING:
@@ -88,6 +82,7 @@ if TYPE_CHECKING:
     from eodag.api.product import Asset, EOProduct  # type: ignore
     from eodag.api.search_result import SearchResult
     from eodag.config import PluginConfig
+    from eodag.types import S3SessionKwargs
     from eodag.types.download_args import DownloadConf
     from eodag.utils import DownloadedCallback, Unpack
 
@@ -100,42 +95,69 @@ class HTTPDownload(Download):
     :param provider: provider name
     :param config: Download plugin configuration:
 
-        * ``config.base_uri`` (str) - (optional) default endpoint url
-        * ``config.extract`` (bool) - (optional) extract downloaded archive or not
-        * ``config.auth_error_code`` (int) - (optional) authentication error code
-        * ``config.dl_url_params`` (dict) - (optional) attitional parameters to send in the request
-        * ``config.archive_depth`` (int) - (optional) level in extracted path tree where to find data
-        * ``config.flatten_top_dirs`` (bool) - (optional) flatten directory structure
-        * ``config.ignore_assets`` (bool) - (optional) ignore assets and download using downloadLink
-        * ``config.order_enabled`` (bool) - (optional) wether order is enabled or not if product is `OFFLINE`
-        * ``config.order_method`` (str) - (optional) HTTP request method, GET (default) or POST
-        * ``config.order_headers`` (dict) - (optional) order request headers
-        * ``config.order_on_response`` (dict) - (optional) edit or add new product properties
-        * ``config.order_status`` (:class:`~eodag.config.PluginConfig.OrderStatus`) - (optional) Order status handling
+        * :attr:`~eodag.config.PluginConfig.type` (``str``) (**mandatory**): ``HTTPDownload``
+        * :attr:`~eodag.config.PluginConfig.base_uri` (``str``): default endpoint url
+        * :attr:`~eodag.config.PluginConfig.method` (``str``): HTTP request method for the download request (``GET`` or
+          ``POST``); default: ``GET``
+        * :attr:`~eodag.config.PluginConfig.extract` (``bool``): if the content of the downloaded file should be
+          extracted; default: ``True``
+        * :attr:`~eodag.config.PluginConfig.auth_error_code` (``int``): which error code is returned in case of an
+          authentication error
+        * :attr:`~eodag.config.PluginConfig.dl_url_params` (``dict[str, Any]``): parameters to be
+          added to the query params of the request
+        * :attr:`~eodag.config.PluginConfig.archive_depth` (``int``): level in extracted path tree where to find data;
+          default: ``1``
+        * :attr:`~eodag.config.PluginConfig.flatten_top_dirs` (``bool``): if the directory structure should be
+          flattened; default: ``True``
+        * :attr:`~eodag.config.PluginConfig.ignore_assets` (``bool``): ignore assets and download using downloadLink;
+          default: ``False``
+        * :attr:`~eodag.config.PluginConfig.timeout` (``int``): time to wait until request timeout in seconds;
+          default: ``5``
+        * :attr:`~eodag.config.PluginConfig.ssl_verify` (``bool``): if the ssl certificates should be verified in
+          requests; default: ``True``
+        * :attr:`~eodag.config.PluginConfig.no_auth_download` (``bool``): if the download should be done without
+          authentication; default: ``True``
+        * :attr:`~eodag.config.PluginConfig.order_enabled` (``bool``): if the product has to be ordered to download it;
+          if this parameter is set to true, a mapping for the orderLink has to be added to the metadata mapping of
+          the search plugin used for the provider; default: ``False``
+        * :attr:`~eodag.config.PluginConfig.order_method` (``str``): HTTP request method for the order request (``GET``
+          or ``POST``); default: ``GET``
+        * :attr:`~eodag.config.PluginConfig.order_headers` (``[dict[str, str]]``): headers to be added to the order
+          request
+        * :attr:`~eodag.config.PluginConfig.order_on_response` (:class:`~eodag.config.PluginConfig.OrderOnResponse`):
+          a typed dictionary containing the key ``metadata_mapping`` which can be used to add new product properties
+          based on the data in response to the order request
+        * :attr:`~eodag.config.PluginConfig.order_status` (:class:`~eodag.config.PluginConfig.OrderStatus`):
+          configuration to handle the order status; contains information which method to use, how the response data is
+          interpreted, which status corresponds to success, ordered and error and what should be done on success.
+        * :attr:`~eodag.config.PluginConfig.products` (``dict[str, dict[str, Any]``): product type specific config; the
+          keys are the product types, the values are dictionaries which can contain the key
+          :attr:`~eodag.config.PluginConfig.extract` to overwrite the provider config for a specific product type
 
     """
 
     def __init__(self, provider: str, config: PluginConfig) -> None:
         super(HTTPDownload, self).__init__(provider, config)
 
-    def order_download(
+    def _order(
         self,
         product: EOProduct,
         auth: Optional[AuthBase] = None,
         **kwargs: Unpack[DownloadConf],
-    ) -> Optional[Dict[str, Any]]:
+    ) -> Optional[dict[str, Any]]:
         """Send product order request.
 
         It will be executed once before the download retry loop, if the product is OFFLINE
         and has `orderLink` in its properties.
         Product ordering can be configured using the following download plugin parameters:
 
-            - **order_enabled**: Wether order is enabled or not (may not use this method
+            - :attr:`~eodag.config.PluginConfig.order_enabled`: Wether order is enabled or not (may not use this method
               if no `orderLink` exists)
 
-            - **order_method**: (optional) HTTP request method, GET (default) or POST
+            - :attr:`~eodag.config.PluginConfig.order_method`: (optional) HTTP request method, GET (default) or POST
 
-            - **order_on_response**: (optional) things to do with obtained order response:
+            - :attr:`~eodag.config.PluginConfig.order_on_response`: (optional) things to do with obtained order
+              response:
 
               - *metadata_mapping*: edit or add new product propoerties properties
 
@@ -154,7 +176,7 @@ class HTTPDownload(Download):
         ssl_verify = getattr(self.config, "ssl_verify", True)
         timeout = getattr(self.config, "timeout", HTTP_REQ_TIMEOUT)
         OrderKwargs = TypedDict(
-            "OrderKwargs", {"json": Dict[str, Union[Any, List[str]]]}, total=False
+            "OrderKwargs", {"json": dict[str, Union[Any, list[str]]]}, total=False
         )
         order_kwargs: OrderKwargs = {}
         if order_method == "POST":
@@ -193,25 +215,20 @@ class HTTPDownload(Download):
                     logger.debug(ordered_message)
                     product.properties["storageStatus"] = STAGING_STATUS
                 except RequestException as e:
-                    if hasattr(e, "response") and (
-                        content := getattr(e.response, "content", None)
-                    ):
-                        error_message = f"{content.decode('utf-8')} - {e}"
-                    else:
-                        error_message = str(e)
-                    logger.warning(
-                        "%s could not be ordered, request returned %s",
-                        product.properties["title"],
-                        error_message,
-                    )
                     self._check_auth_exception(e)
+                    msg = f'{product.properties["title"]} could not be ordered'
+                    if e.response is not None and e.response.status_code == 400:
+                        raise ValidationError.from_error(e, msg) from e
+                    else:
+                        raise DownloadError.from_error(e, msg) from e
+
                 return self.order_response_process(response, product)
         except requests.exceptions.Timeout as exc:
             raise TimeOutError(exc, timeout=timeout) from exc
 
     def order_response_process(
         self, response: Response, product: EOProduct
-    ) -> Optional[Dict[str, Any]]:
+    ) -> Optional[dict[str, Any]]:
         """Process order response
 
         :param response: The order response
@@ -246,7 +263,7 @@ class HTTPDownload(Download):
 
         return json_response
 
-    def order_download_status(
+    def _order_status(
         self,
         product: EOProduct,
         auth: Optional[AuthBase] = None,
@@ -256,7 +273,7 @@ class HTTPDownload(Download):
         It will be executed before each download retry.
         Product order status request can be configured using the following download plugin parameters:
 
-            - **order_status**: :class:`~eodag.config.PluginConfig.OrderStatus`
+            - :attr:`~eodag.config.PluginConfig.order_status`: :class:`~eodag.config.PluginConfig.OrderStatus`
 
         Product properties used for order status:
 
@@ -275,7 +292,7 @@ class HTTPDownload(Download):
         def _request(
             url: str,
             method: str = "GET",
-            headers: Optional[Dict[str, Any]] = None,
+            headers: Optional[dict[str, Any]] = None,
             json: Optional[Any] = None,
             timeout: int = HTTP_REQ_TIMEOUT,
         ) -> Response:
@@ -311,7 +328,7 @@ class HTTPDownload(Download):
             except requests.exceptions.Timeout as exc:
                 raise TimeOutError(exc, timeout=timeout) from exc
 
-        status_request: Dict[str, Any] = status_config.get("request", {})
+        status_request: dict[str, Any] = status_config.get("request", {})
         status_request_method = str(status_request.get("method", "GET")).upper()
 
         if status_request_method == "POST":
@@ -328,8 +345,8 @@ class HTTPDownload(Download):
 
         # check header for success before full status request
         skip_parsing_status_response = False
-        status_dict: Dict[str, Any] = {}
-        config_on_success: Dict[str, Any] = status_config.get("on_success", {})
+        status_dict: dict[str, Any] = {}
+        config_on_success: dict[str, Any] = status_config.get("on_success", {})
         on_success_mm = config_on_success.get("metadata_mapping", {})
 
         status_response_content_needed = (
@@ -373,13 +390,11 @@ class HTTPDownload(Download):
                     # success and no need to get status response content
                     skip_parsing_status_response = True
             except RequestException as e:
-                raise DownloadError(
-                    "%s order status could not be checked, request returned %s"
-                    % (
-                        product.properties["title"],
-                        e,
-                    )
-                ) from e
+                msg = f'{product.properties["title"]} order status could not be checked'
+                if e.response is not None and e.response.status_code == 400:
+                    raise ValidationError.from_error(e, msg) from e
+                else:
+                    raise DownloadError.from_error(e, msg) from e
 
         if not skip_parsing_status_response:
             # status request
@@ -415,13 +430,13 @@ class HTTPDownload(Download):
             product.properties["orderStatus"] = status_dict.get("status")
 
             # handle status error
-            errors: Dict[str, Any] = status_config.get("error", {})
+            errors: dict[str, Any] = status_config.get("error", {})
             if errors and errors.items() <= status_dict.items():
                 raise DownloadError(
                     f"Provider {product.provider} returned: {status_dict.get('error_message', status_message)}"
                 )
 
-        success_status: Dict[str, Any] = status_config.get("success", {}).get("status")
+        success_status: dict[str, Any] = status_config.get("success", {}).get("status")
         # if not success
         if (success_status and success_status != status_dict.get("status")) or (
             success_code and success_code != response.status_code
@@ -539,10 +554,10 @@ class HTTPDownload(Download):
     def download(
         self,
         product: EOProduct,
-        auth: Optional[Union[AuthBase, Dict[str, str]]] = None,
+        auth: Optional[Union[AuthBase, S3SessionKwargs]] = None,
         progress_callback: Optional[ProgressCallback] = None,
-        wait: int = DEFAULT_DOWNLOAD_WAIT,
-        timeout: int = DEFAULT_DOWNLOAD_TIMEOUT,
+        wait: float = DEFAULT_DOWNLOAD_WAIT,
+        timeout: float = DEFAULT_DOWNLOAD_TIMEOUT,
         **kwargs: Unpack[DownloadConf],
     ) -> Optional[str]:
         """Download a product using HTTP protocol.
@@ -560,13 +575,6 @@ class HTTPDownload(Download):
             )
             progress_callback = ProgressCallback(disable=True)
 
-        output_extension = getattr(self.config, "products", {}).get(
-            product.product_type, {}
-        ).get("output_extension", None) or getattr(
-            self.config, "output_extension", ".zip"
-        )
-        kwargs["output_extension"] = kwargs.get("output_extension", output_extension)
-
         fs_path, record_filename = self._prepare_download(
             product,
             progress_callback=progress_callback,
@@ -585,7 +593,7 @@ class HTTPDownload(Download):
             try:
                 fs_path = self._download_assets(
                     product,
-                    fs_path.replace(".zip", ""),
+                    fs_path,
                     record_filename,
                     auth,
                     progress_callback,
@@ -602,82 +610,64 @@ class HTTPDownload(Download):
 
         url = product.remote_location
 
-        @self._download_retry(product, wait, timeout)
+        @self._order_download_retry(product, wait, timeout)
         def download_request(
             product: EOProduct,
             auth: AuthBase,
             progress_callback: ProgressCallback,
-            wait: int,
-            timeout: int,
+            wait: float,
+            timeout: float,
             **kwargs: Unpack[DownloadConf],
-        ) -> None:
-            chunks = self._stream_download(product, auth, progress_callback, **kwargs)
+        ) -> os.PathLike:
             is_empty = True
+            chunk_iterator = self._stream_download(
+                product, auth, progress_callback, **kwargs
+            )
+            if fs_path is not None:
+                ext = Path(product.filename).suffix
+                path = Path(fs_path).with_suffix(ext)
+
+                with open(path, "wb") as fhandle:
+                    for chunk in chunk_iterator:
+                        is_empty = False
+                        progress_callback(len(chunk))
+                        fhandle.write(chunk)
+                self.stream.close()  # Closing response stream
 
-            with open(fs_path, "wb") as fhandle:
-                for chunk in chunks:
-                    is_empty = False
-                    fhandle.write(chunk)
+                if is_empty:
+                    raise DownloadError(f"product {product.properties['id']} is empty")
 
-            if is_empty:
-                raise DownloadError(f"product {product.properties['id']} is empty")
+                return path
+            else:
+                raise DownloadError(
+                    f"download of product {product.properties['id']} failed"
+                )
 
-        download_request(product, auth, progress_callback, wait, timeout, **kwargs)
+        path = download_request(
+            product, auth, progress_callback, wait, timeout, **kwargs
+        )
 
         with open(record_filename, "w") as fh:
             fh.write(url)
         logger.debug("Download recorded in %s", record_filename)
 
-        # Check that the downloaded file is really a zip file
-        if not zipfile.is_zipfile(fs_path) and output_extension == ".zip":
-            logger.warning(
-                "Downloaded product is not a Zip File. Please check its file type before using it"
-            )
-            new_fs_path = os.path.join(
-                os.path.dirname(fs_path),
-                sanitize(product.properties["title"]),
-            )
-            if os.path.isfile(fs_path) and not tarfile.is_tarfile(fs_path):
-                if not os.path.isdir(new_fs_path):
-                    os.makedirs(new_fs_path)
-                shutil.move(fs_path, new_fs_path)
-                file_path = os.path.join(new_fs_path, os.path.basename(fs_path))
-                new_file_path = file_path[: file_path.index(".zip")]
-                shutil.move(file_path, new_file_path)
-            # in the case where the outputs extension has not been set
-            # to ".tar" in the product type nor provider configuration
-            elif tarfile.is_tarfile(fs_path):
-                if not new_fs_path.endswith(".tar"):
-                    new_fs_path += ".tar"
-                shutil.move(fs_path, new_fs_path)
-                kwargs["output_extension"] = ".tar"
-                product_path = self._finalize(
-                    new_fs_path,
-                    progress_callback=progress_callback,
-                    **kwargs,
-                )
-                product.location = path_to_uri(product_path)
-                return product_path
-            else:
-                # not a file (dir with zip extension)
-                shutil.move(fs_path, new_fs_path)
-            product.location = path_to_uri(new_fs_path)
-            return new_fs_path
-
-        if os.path.isfile(fs_path) and not (
-            zipfile.is_zipfile(fs_path) or tarfile.is_tarfile(fs_path)
+        if os.path.isfile(path) and not (
+            zipfile.is_zipfile(path) or tarfile.is_tarfile(path)
         ):
             new_fs_path = os.path.join(
-                os.path.dirname(fs_path),
+                os.path.dirname(path),
                 sanitize(product.properties["title"]),
             )
+            if os.path.isfile(new_fs_path):
+                rename_with_version(new_fs_path)
             if not os.path.isdir(new_fs_path):
                 os.makedirs(new_fs_path)
-            shutil.move(fs_path, new_fs_path)
+            shutil.move(path, new_fs_path)
             product.location = path_to_uri(new_fs_path)
             return new_fs_path
+
         product_path = self._finalize(
-            fs_path,
+            str(path),
             progress_callback=progress_callback,
             **kwargs,
         )
@@ -718,28 +708,19 @@ class HTTPDownload(Download):
                     ext = guess_extension(content_type)
                     if ext:
                         filename += ext
-                else:
-                    output_extension: Optional[str] = (
-                        getattr(self.config, "products", {})
-                        .get(product.product_type, {})
-                        .get("output_extension")
-                    )
-                    if output_extension:
-                        filename += output_extension
-
         return filename
 
     def _stream_download_dict(
         self,
         product: EOProduct,
-        auth: Optional[Union[AuthBase, Dict[str, str]]] = None,
+        auth: Optional[Union[AuthBase, S3SessionKwargs]] = None,
         progress_callback: Optional[ProgressCallback] = None,
-        wait: int = DEFAULT_DOWNLOAD_WAIT,
-        timeout: int = DEFAULT_DOWNLOAD_TIMEOUT,
+        wait: float = DEFAULT_DOWNLOAD_WAIT,
+        timeout: float = DEFAULT_DOWNLOAD_TIMEOUT,
         **kwargs: Unpack[DownloadConf],
     ) -> StreamResponse:
         r"""
-        Returns dictionnary of :class:`~fastapi.responses.StreamingResponse` keyword-arguments.
+        Returns dictionary of :class:`~fastapi.responses.StreamingResponse` keyword-arguments.
         It contains a generator to streamed download chunks and the response headers.
 
         :param product: The EO product to download
@@ -752,7 +733,7 @@ class HTTPDownload(Download):
                         and `dl_url_params` (dict) can be provided as additional kwargs
                         and will override any other values defined in a configuration
                         file or with environment variables.
-        :returns: Dictionnary of :class:`~fastapi.responses.StreamingResponse` keyword-arguments
+        :returns: Dictionary of :class:`~fastapi.responses.StreamingResponse` keyword-arguments
         """
         if auth is not None and not isinstance(auth, AuthBase):
             raise MisconfiguredError(f"Incompatible auth plugin: {type(auth)}")
@@ -791,13 +772,17 @@ class HTTPDownload(Download):
                     )
 
                 else:
+                    # get first chunk to check if it does not contain an error (if it does, that error will be raised)
+                    first_chunks_tuple = next(chunks_tuples)
                     outputs_filename = (
                         sanitize(product.properties["title"])
                         if "title" in product.properties
                         else sanitize(product.properties.get("id", "download"))
                     )
                     return StreamResponse(
-                        content=stream_zip(chunks_tuples),
+                        content=stream_zip(
+                            chain(iter([first_chunks_tuple]), chunks_tuples)
+                        ),
                         media_type="application/zip",
                         headers={
                             "content-disposition": f"attachment; filename={outputs_filename}.zip",
@@ -809,17 +794,20 @@ class HTTPDownload(Download):
                 else:
                     pass
 
-        chunks = self._stream_download(product, auth, progress_callback, **kwargs)
+        chunk_iterator = self._stream_download(
+            product, auth, progress_callback, **kwargs
+        )
+
         # start reading chunks to set product.headers
         try:
-            first_chunk = next(chunks)
+            first_chunk = next(chunk_iterator)
         except StopIteration:
             # product is empty file
             logger.error("product %s is empty", product.properties["id"])
             raise NotAvailableError(f"product {product.properties['id']} is empty")
 
         return StreamResponse(
-            content=chain(iter([first_chunk]), chunks),
+            content=chain(iter([first_chunk]), chunk_iterator),
             headers=product.headers,
         )
 
@@ -837,12 +825,9 @@ class HTTPDownload(Download):
             and e.response.status_code in auth_errors
         ):
             raise AuthenticationError(
-                "HTTP Error %s returned, %s\nPlease check your credentials for %s"
-                % (
-                    e.response.status_code,
-                    response_text,
-                    self.provider,
-                )
+                f"Please check your credentials for {self.provider}.",
+                f"HTTP Error {e.response.status_code} returned.",
+                response_text,
             )
 
     def _process_exception(
@@ -880,6 +865,44 @@ class HTTPDownload(Download):
             else:
                 logger.error("Error while getting resource :\n%s", tb.format_exc())
 
+    def _order_request(
+        self,
+        product: EOProduct,
+        auth: Optional[AuthBase],
+    ) -> None:
+        if (
+            "orderLink" in product.properties
+            and product.properties.get("storageStatus") == OFFLINE_STATUS
+            and not product.properties.get("orderStatus")
+        ):
+            self._order(product=product, auth=auth)
+
+        if (
+            product.properties.get("orderStatusLink", None)
+            and product.properties.get("storageStatus") != ONLINE_STATUS
+        ):
+            self._order_status(product=product, auth=auth)
+
+    def order(
+        self,
+        product: EOProduct,
+        auth: Optional[Union[AuthBase, S3SessionKwargs]] = None,
+        wait: float = DEFAULT_DOWNLOAD_WAIT,
+        timeout: float = DEFAULT_DOWNLOAD_TIMEOUT,
+    ) -> None:
+        """
+        Order product and poll to check its status
+
+        :param product: The EO product to download
+        :param auth: (optional) authenticated object
+        :param wait: (optional) Wait time in minutes between two order status check
+        :param timeout: (optional) Maximum time in minutes before stop checking
+                        order status
+        """
+        self._order_download_retry(product, wait, timeout)(self._order_request)(
+            product, auth
+        )
+
     def _stream_download(
         self,
         product: EOProduct,
@@ -888,8 +911,9 @@ class HTTPDownload(Download):
         **kwargs: Unpack[DownloadConf],
     ) -> Iterator[Any]:
         """
-        fetches a zip file containing the assets of a given product as a stream
+        Fetches a zip file containing the assets of a given product as a stream
         and returns a generator yielding the chunks of the file
+
         :param product: product for which the assets should be downloaded
         :param auth: The configuration of a plugin of type Authentication
         :param progress_callback: A method or a callable object
@@ -903,19 +927,11 @@ class HTTPDownload(Download):
             logger.info("Progress bar unavailable, please call product.download()")
             progress_callback = ProgressCallback(disable=True)
 
-        ordered_message = ""
-        if (
-            "orderLink" in product.properties
-            and product.properties.get("storageStatus") == OFFLINE_STATUS
-            and not product.properties.get("orderStatus")
-        ):
-            self.order_download(product=product, auth=auth)
+        ssl_verify = getattr(self.config, "ssl_verify", True)
 
-        if (
-            product.properties.get("orderStatusLink", None)
-            and product.properties.get("storageStatus") != ONLINE_STATUS
-        ):
-            self.order_download_status(product=product, auth=auth)
+        ordered_message = ""
+        # retry handled at download level
+        self._order_request(product, auth)
 
         params = kwargs.pop("dl_url_params", None) or getattr(
             self.config, "dl_url_params", {}
@@ -933,7 +949,7 @@ class HTTPDownload(Download):
             if not query_dict and parts.query:
                 query_dict = geojson.loads(parts.query)
             req_url = parts._replace(query="").geturl()
-            req_kwargs: Dict[str, Any] = {"json": query_dict} if query_dict else {}
+            req_kwargs: dict[str, Any] = {"json": query_dict} if query_dict else {}
         else:
             req_url = url
             req_kwargs = {}
@@ -945,7 +961,7 @@ class HTTPDownload(Download):
             auth = None
 
         s = requests.Session()
-        with s.request(
+        self.stream = s.request(
             req_method,
             req_url,
             stream=True,
@@ -953,56 +969,55 @@ class HTTPDownload(Download):
             params=params,
             headers=USER_AGENT,
             timeout=DEFAULT_STREAM_REQUESTS_TIMEOUT,
+            verify=ssl_verify,
             **req_kwargs,
-        ) as self.stream:
-            try:
-                self.stream.raise_for_status()
-            except requests.exceptions.Timeout as exc:
-                raise TimeOutError(
-                    exc, timeout=DEFAULT_STREAM_REQUESTS_TIMEOUT
-                ) from exc
-            except RequestException as e:
-                self._process_exception(e, product, ordered_message)
-            else:
-                # check if product was ordered
-
-                if getattr(
-                    self.stream, "status_code", None
-                ) is not None and self.stream.status_code == getattr(
-                    self.config, "order_status", {}
-                ).get(
-                    "ordered", {}
-                ).get(
-                    "http_code"
-                ):
-                    product.properties["storageStatus"] = "ORDERED"
-                    self._process_exception(None, product, ordered_message)
-                stream_size = self._check_stream_size(product) or None
-
-                product.headers = self.stream.headers
-                filename = self._check_product_filename(product) or None
-                product.headers[
-                    "content-disposition"
-                ] = f"attachment; filename={filename}"
-                content_type = product.headers.get("Content-Type")
-                guessed_content_type = (
-                    guess_file_type(filename) if filename and not content_type else None
-                )
-                if guessed_content_type is not None:
-                    product.headers["Content-Type"] = guessed_content_type
+        )
+        try:
+            self.stream.raise_for_status()
+        except requests.exceptions.Timeout as exc:
+            raise TimeOutError(exc, timeout=DEFAULT_STREAM_REQUESTS_TIMEOUT) from exc
+        except RequestException as e:
+            self._process_exception(e, product, ordered_message)
+            raise DownloadError(
+                f"download of {product.properties['id']} is empty"
+            ) from e
+        else:
+            # check if product was ordered
+
+            if getattr(
+                self.stream, "status_code", None
+            ) is not None and self.stream.status_code == getattr(
+                self.config, "order_status", {}
+            ).get(
+                "ordered", {}
+            ).get(
+                "http_code"
+            ):
+                product.properties["storageStatus"] = "ORDERED"
+                self._process_exception(None, product, ordered_message)
+            stream_size = self._check_stream_size(product) or None
+
+            product.headers = self.stream.headers
+            filename = self._check_product_filename(product)
+            product.headers["content-disposition"] = f"attachment; filename={filename}"
+            content_type = product.headers.get("Content-Type")
+            guessed_content_type = (
+                guess_file_type(filename) if filename and not content_type else None
+            )
+            if guessed_content_type is not None:
+                product.headers["Content-Type"] = guessed_content_type
 
-                progress_callback.reset(total=stream_size)
-                for chunk in self.stream.iter_content(chunk_size=64 * 1024):
-                    if chunk:
-                        progress_callback(len(chunk))
-                        yield chunk
+            progress_callback.reset(total=stream_size)
+
+            product.filename = filename
+            return self.stream.iter_content(chunk_size=64 * 1024)
 
     def _stream_download_assets(
         self,
         product: EOProduct,
         auth: Optional[AuthBase] = None,
         progress_callback: Optional[ProgressCallback] = None,
-        assets_values: List[Asset] = [],
+        assets_values: list[Asset] = [],
         **kwargs: Unpack[DownloadConf],
     ) -> Iterator[Any]:
         if progress_callback is None:
@@ -1055,6 +1070,16 @@ class HTTPDownload(Download):
             "flatten_top_dirs", getattr(self.config, "flatten_top_dirs", True)
         )
         ssl_verify = getattr(self.config, "ssl_verify", True)
+        matching_url = (
+            getattr(product.downloader_auth.config, "matching_url", "")
+            if product.downloader_auth
+            else ""
+        )
+        matching_conf = (
+            getattr(product.downloader_auth.config, "matching_conf", None)
+            if product.downloader_auth
+            else None
+        )
 
         # loop for assets download
         for asset in assets_values:
@@ -1063,11 +1088,16 @@ class HTTPDownload(Download):
                     f"Local asset detected. Download skipped for {asset['href']}"
                 )
                 continue
-
+            if matching_conf or (
+                matching_url and re.match(matching_url, asset["href"])
+            ):
+                auth_object = auth
+            else:
+                auth_object = None
             with requests.get(
                 asset["href"],
                 stream=True,
-                auth=auth,
+                auth=auth_object,
                 params=params,
                 headers=USER_AGENT,
                 timeout=DEFAULT_STREAM_REQUESTS_TIMEOUT,
@@ -1080,8 +1110,7 @@ class HTTPDownload(Download):
                         exc, timeout=DEFAULT_STREAM_REQUESTS_TIMEOUT
                     ) from exc
                 except RequestException as e:
-                    raise_errors = True if len(assets_values) == 1 else False
-                    self._handle_asset_exception(e, asset, raise_errors=raise_errors)
+                    self._handle_asset_exception(e, asset)
                 else:
                     asset_rel_path = (
                         asset.rel_path.replace(assets_common_subdir, "").strip(os.sep)
@@ -1239,33 +1268,28 @@ class HTTPDownload(Download):
 
         return fs_dir_path
 
-    def _handle_asset_exception(
-        self, e: RequestException, asset: Asset, raise_errors: bool = False
-    ) -> None:
+    def _handle_asset_exception(self, e: RequestException, asset: Asset) -> None:
         # check if error is identified as auth_error in provider conf
         auth_errors = getattr(self.config, "auth_error_code", [None])
         if not isinstance(auth_errors, list):
             auth_errors = [auth_errors]
         if e.response is not None and e.response.status_code in auth_errors:
             raise AuthenticationError(
-                "HTTP Error %s returned, %s\nPlease check your credentials for %s"
-                % (
-                    e.response.status_code,
-                    e.response.text.strip(),
-                    self.provider,
-                )
+                f"Please check your credentials for {self.provider}.",
+                f"HTTP Error {e.response.status_code} returned.",
+                e.response.text.strip(),
             )
-        elif raise_errors:
-            raise DownloadError(e)
         else:
-            logger.warning("Unexpected error: %s" % e)
-            logger.warning("Skipping %s" % asset["href"])
+            logger.error(
+                "Unexpected error at download of asset %s: %s", asset["href"], e
+            )
+            raise DownloadError(e)
 
     def _get_asset_sizes(
         self,
-        assets_values: List[Asset],
+        assets_values: list[Asset],
         auth: Optional[AuthBase],
-        params: Optional[Dict[str, str]],
+        params: Optional[dict[str, str]],
         zipped: bool = False,
     ) -> int:
         total_size = 0
@@ -1338,11 +1362,11 @@ class HTTPDownload(Download):
     def download_all(
         self,
         products: SearchResult,
-        auth: Optional[Union[AuthBase, Dict[str, str]]] = None,
+        auth: Optional[Union[AuthBase, S3SessionKwargs]] = None,
         downloaded_callback: Optional[DownloadedCallback] = None,
         progress_callback: Optional[ProgressCallback] = None,
-        wait: int = DEFAULT_DOWNLOAD_WAIT,
-        timeout: int = DEFAULT_DOWNLOAD_TIMEOUT,
+        wait: float = DEFAULT_DOWNLOAD_WAIT,
+        timeout: float = DEFAULT_DOWNLOAD_TIMEOUT,
         **kwargs: Unpack[DownloadConf],
     ):
         """