eodag 4.0.0a4__py3-none-any.whl → 4.0.0a5__py3-none-any.whl

eodag/api/core.py

@@ -87,6 +87,7 @@ from eodag.utils.free_text_search import compile_free_text_query
 from eodag.utils.stac_reader import fetch_stac_items
 
 if TYPE_CHECKING:
+    from concurrent.futures import ThreadPoolExecutor
     from shapely.geometry.base import BaseGeometry
 
     from eodag.api.product import EOProduct
@@ -452,6 +453,9 @@ class EODataAccessGateway:
             if locations_conf_path is None:
                 locations_conf_path = os.path.join(self.conf_dir, "locations.yml")
                 if not os.path.isfile(locations_conf_path):
+                    # Ensure the directory exists
+                    os.makedirs(os.path.dirname(locations_conf_path), exist_ok=True)
+
                     # copy locations conf file and replace path example
                     locations_conf_template = str(
                         res_files("eodag") / "resources" / "locations_conf_template.yml"
@@ -1919,6 +1923,7 @@ class EODataAccessGateway:
         search_result: SearchResult,
         downloaded_callback: Optional[DownloadedCallback] = None,
         progress_callback: Optional[ProgressCallback] = None,
+        executor: Optional[ThreadPoolExecutor] = None,
         wait: float = DEFAULT_DOWNLOAD_WAIT,
         timeout: float = DEFAULT_DOWNLOAD_TIMEOUT,
         **kwargs: Unpack[DownloadConf],
@@ -1936,6 +1941,8 @@ class EODataAccessGateway:
                                   size as inputs and handle progress bar
                                   creation and update to give the user a
                                   feedback on the download progress
+        :param executor: (optional) An executor to download EO products of ``search_result`` in parallel
+                                    which will also be reused to download assets of these products in parallel.
         :param wait: (optional) If download fails, wait time in minutes between
                      two download tries of the same product
         :param timeout: (optional) If download fails, maximum time in minutes
@@ -1956,8 +1963,7 @@ class EODataAccessGateway:
         paths = []
         if search_result:
             logger.info("Downloading %s products", len(search_result))
-            # Get download plugin using first product assuming product from several provider
-            # aren't mixed into a search result
+            # Get download plugin using first product assuming all plugins use base.Download.download_all
             download_plugin = self._plugins_manager.get_download_plugin(
                 search_result[0]
             )
@@ -1965,6 +1971,7 @@ class EODataAccessGateway:
                 search_result,
                 downloaded_callback=downloaded_callback,
                 progress_callback=progress_callback,
+                executor=executor,
                 wait=wait,
                 timeout=timeout,
                 **kwargs,
@@ -2026,6 +2033,7 @@ class EODataAccessGateway:
         self,
         product: EOProduct,
         progress_callback: Optional[ProgressCallback] = None,
+        executor: Optional[ThreadPoolExecutor] = None,
         wait: float = DEFAULT_DOWNLOAD_WAIT,
         timeout: float = DEFAULT_DOWNLOAD_TIMEOUT,
         **kwargs: Unpack[DownloadConf],
@@ -2056,6 +2064,8 @@ class EODataAccessGateway:
                                   size as inputs and handle progress bar
                                   creation and update to give the user a
                                   feedback on the download progress
+        :param executor: (optional) An executor to download assets of ``product`` in parallel if it has any. If ``None``
+                         , a default executor will be created
         :param wait: (optional) If download fails, wait time in minutes between
                     two download tries
         :param timeout: (optional) If download fails, maximum time in minutes
@@ -2080,7 +2090,11 @@ class EODataAccessGateway:
             return uri_to_path(product.location)
         self._setup_downloader(product)
         path = product.download(
-            progress_callback=progress_callback, wait=wait, timeout=timeout, **kwargs
+            progress_callback=progress_callback,
+            executor=executor,
+            wait=wait,
+            timeout=timeout,
+            **kwargs,
         )
 
         return path

eodag/api/product/_product.py

@@ -61,6 +61,7 @@ from eodag.utils.exceptions import DownloadError, MisconfiguredError, Validation
 from eodag.utils.repr import dict_to_html_table
 
 if TYPE_CHECKING:
+    from concurrent.futures import ThreadPoolExecutor
     from shapely.geometry.base import BaseGeometry
 
     from eodag.api.product.drivers.base import DatasetDriver
@@ -122,6 +123,8 @@ class EOProduct:
     search_kwargs: Any
     #: Datetime for download next try
     next_try: datetime
+    #: Stream for requests
+    _stream: requests.Response
 
     def __init__(
         self, provider: str, properties: dict[str, Any], **kwargs: Any
@@ -337,6 +340,7 @@ class EOProduct:
     def download(
         self,
         progress_callback: Optional[ProgressCallback] = None,
+        executor: Optional[ThreadPoolExecutor] = None,
         wait: float = DEFAULT_DOWNLOAD_WAIT,
         timeout: float = DEFAULT_DOWNLOAD_TIMEOUT,
         **kwargs: Unpack[DownloadConf],
@@ -353,6 +357,8 @@ class EOProduct:
                                   size as inputs and handle progress bar
                                   creation and update to give the user a
                                   feedback on the download progress
+        :param executor: (optional) An executor to download assets of the product in parallel if it has any. If ``None``
+                         , a default executor will be created
         :param wait: (optional) If download fails, wait time in minutes between
                      two download tries
         :param timeout: (optional) If download fails, maximum time in minutes
@@ -377,17 +383,26 @@ class EOProduct:
         )
 
         progress_callback, close_progress_callback = self._init_progress_bar(
-            progress_callback
+            progress_callback, executor
         )
+
         fs_path = self.downloader.download(
             self,
             auth=auth,
             progress_callback=progress_callback,
+            executor=executor,
             wait=wait,
             timeout=timeout,
             **kwargs,
         )
 
+        # shutdown executor if it was not created during parallel product downloads
+        if (
+            executor is not None
+            and executor._thread_name_prefix != "eodag-download-all"
+        ):
+            executor.shutdown(wait=True)
+
         # close progress bar if needed
         if close_progress_callback:
             progress_callback.close()
@@ -408,15 +423,22 @@ class EOProduct:
         return fs_path
 
     def _init_progress_bar(
-        self, progress_callback: Optional[ProgressCallback]
+        self,
+        progress_callback: Optional[ProgressCallback],
+        executor: Optional[ThreadPoolExecutor],
     ) -> tuple[ProgressCallback, bool]:
+        # determine position of the progress bar with a counter of executor passings
+        # to avoid bar overwriting in case of parallel downloads
+        count = executor._counter() if executor is not None else 1  # type: ignore
+
         # progress bar init
         if progress_callback is None:
-            progress_callback = ProgressCallback(position=1)
+            progress_callback = ProgressCallback(position=count)
             # one shot progress callback to close after download
             close_progress_callback = True
         else:
             close_progress_callback = False
+            progress_callback.pos = count
             # update units as bar may have been previously used for extraction
             progress_callback.unit = "B"
             progress_callback.unit_scale = True

eodag/cli.py

@@ -48,6 +48,7 @@ from typing import TYPE_CHECKING, Any, Callable, Mapping, Optional
 from urllib.parse import parse_qs
 
 import click
+from concurrent.futures import ThreadPoolExecutor
 
 from eodag.api.collection import CollectionsList
 from eodag.api.core import EODataAccessGateway, SearchResult
@@ -556,6 +557,11 @@ Examples:
     type=click.Path(dir_okay=True, file_okay=False),
     help="Products or quicklooks download directory (Default: local temporary directory)",
 )
+@click.option(
+    "--max-workers",
+    type=int,
+    help="The maximum number of workers to use for downloading products and assets in parallel",
+)
 @click.pass_context
 def download(ctx: Context, **kwargs: Any) -> None:
     """Download a bunch of products from a serialized search result"""
@@ -601,7 +607,10 @@ def download(ctx: Context, **kwargs: Any) -> None:
 
     else:
         # Download products
-        downloaded_files = satim_api.download_all(search_results, output_dir=output_dir)
+        executor = ThreadPoolExecutor(max_workers=kwargs.pop("max_workers"))
+        downloaded_files = satim_api.download_all(
+            search_results, output_dir=output_dir, executor=executor
+        )
         if downloaded_files and len(downloaded_files) > 0:
             for downloaded_file in downloaded_files:
                 if downloaded_file is None:

eodag/plugins/apis/ecmwf.py

@@ -48,6 +48,7 @@ from eodag.utils.logging import get_logging_verbose
 if TYPE_CHECKING:
     from typing import Any, Optional, Union
 
+    from concurrent.futures import ThreadPoolExecutor
     from mypy_boto3_s3 import S3ServiceResource
     from requests.auth import AuthBase
 
@@ -55,7 +56,7 @@ if TYPE_CHECKING:
     from eodag.api.search_result import SearchResult
     from eodag.config import PluginConfig
     from eodag.types.download_args import DownloadConf
-    from eodag.utils import DownloadedCallback, ProgressCallback, Unpack
+    from eodag.utils import ProgressCallback, Unpack
 
 
 logger = logging.getLogger("eodag.apis.ecmwf")
@@ -185,6 +186,7 @@ class EcmwfApi(Api, ECMWFSearch):
         product: EOProduct,
         auth: Optional[Union[AuthBase, S3ServiceResource]] = None,
         progress_callback: Optional[ProgressCallback] = None,
+        executor: Optional[ThreadPoolExecutor] = None,
         wait: float = DEFAULT_DOWNLOAD_WAIT,
         timeout: float = DEFAULT_DOWNLOAD_TIMEOUT,
         **kwargs: Unpack[DownloadConf],
@@ -269,29 +271,6 @@ class EcmwfApi(Api, ECMWFSearch):
         product.location = path_to_uri(product_path)
         return product_path
 
-    def download_all(
-        self,
-        products: SearchResult,
-        auth: Optional[Union[AuthBase, S3ServiceResource]] = None,
-        downloaded_callback: Optional[DownloadedCallback] = None,
-        progress_callback: Optional[ProgressCallback] = None,
-        wait: float = DEFAULT_DOWNLOAD_WAIT,
-        timeout: float = DEFAULT_DOWNLOAD_TIMEOUT,
-        **kwargs: Unpack[DownloadConf],
-    ) -> list[str]:
-        """
-        Download all using parent (base plugin) method
-        """
-        return super(EcmwfApi, self).download_all(
-            products,
-            auth=auth,
-            downloaded_callback=downloaded_callback,
-            progress_callback=progress_callback,
-            wait=wait,
-            timeout=timeout,
-            **kwargs,
-        )
-
     def clear(self) -> None:
         """Clear search context"""
         pass

eodag/plugins/apis/usgs.py

@@ -57,12 +57,13 @@ from eodag.utils.exceptions import (
 )
 
 if TYPE_CHECKING:
+    from concurrent.futures import ThreadPoolExecutor
     from mypy_boto3_s3 import S3ServiceResource
     from requests.auth import AuthBase
 
     from eodag.config import PluginConfig
     from eodag.types.download_args import DownloadConf
-    from eodag.utils import DownloadedCallback, Unpack
+    from eodag.utils import Unpack
 
 logger = logging.getLogger("eodag.apis.usgs")
 
@@ -312,6 +313,7 @@ class UsgsApi(Api):
         product: EOProduct,
         auth: Optional[Union[AuthBase, S3ServiceResource]] = None,
         progress_callback: Optional[ProgressCallback] = None,
+        executor: Optional[ThreadPoolExecutor] = None,
         wait: float = DEFAULT_DOWNLOAD_WAIT,
         timeout: float = DEFAULT_DOWNLOAD_TIMEOUT,
         **kwargs: Unpack[DownloadConf],
@@ -477,26 +479,3 @@ class UsgsApi(Api):
             shutil.move(fs_path, new_fs_path)
             product.location = path_to_uri(new_fs_path)
             return new_fs_path
-
-    def download_all(
-        self,
-        products: SearchResult,
-        auth: Optional[Union[AuthBase, S3ServiceResource]] = None,
-        downloaded_callback: Optional[DownloadedCallback] = None,
-        progress_callback: Optional[ProgressCallback] = None,
-        wait: float = DEFAULT_DOWNLOAD_WAIT,
-        timeout: float = DEFAULT_DOWNLOAD_TIMEOUT,
-        **kwargs: Unpack[DownloadConf],
-    ) -> list[str]:
-        """
-        Download all using parent (base plugin) method
-        """
-        return super(UsgsApi, self).download_all(
-            products,
-            auth=auth,
-            downloaded_callback=downloaded_callback,
-            progress_callback=progress_callback,
-            wait=wait,
-            timeout=timeout,
-            **kwargs,
-        )

eodag/plugins/download/aws.py

@@ -25,7 +25,9 @@ from typing import TYPE_CHECKING, Any, Literal, Optional, Union, cast
 
 import boto3
 import requests
+from boto3.s3.transfer import TransferConfig
 from botocore.exceptions import ClientError
+from concurrent.futures import ThreadPoolExecutor, as_completed
 from lxml import etree
 from requests.auth import AuthBase
 
@@ -34,7 +36,7 @@ from eodag.api.product.metadata_mapping import (
     properties_from_json,
     properties_from_xml,
 )
-from eodag.plugins.authentication.aws_auth import raise_if_auth_error
+from eodag.plugins.authentication.aws_auth import AwsAuth, raise_if_auth_error
 from eodag.plugins.download.base import Download
 from eodag.utils import (
     DEFAULT_DOWNLOAD_TIMEOUT,
@@ -65,10 +67,9 @@ if TYPE_CHECKING:
     from mypy_boto3_s3.client import S3Client
 
     from eodag.api.product import EOProduct
-    from eodag.api.search_result import SearchResult
     from eodag.config import PluginConfig
     from eodag.types.download_args import DownloadConf
-    from eodag.utils import DownloadedCallback, Unpack
+    from eodag.utils import Unpack
 
 
 logger = logging.getLogger("eodag.download.aws")
@@ -227,6 +228,7 @@ class AwsDownload(Download):
         product: EOProduct,
         auth: Optional[Union[AuthBase, S3ServiceResource]] = None,
         progress_callback: Optional[ProgressCallback] = None,
+        executor: Optional[ThreadPoolExecutor] = None,
         wait: float = DEFAULT_DOWNLOAD_WAIT,
         timeout: float = DEFAULT_DOWNLOAD_TIMEOUT,
         **kwargs: Unpack[DownloadConf],
@@ -246,6 +248,7 @@ class AwsDownload(Download):
                                   size as inputs and handle progress bar
                                   creation and update to give the user a
                                   feedback on the download progress
+        :param executor: (optional) An executor to download assets of ``product`` in parallel if it has any
         :param kwargs: `output_dir` (str), `extract` (bool), `delete_archive` (bool)
                         and `dl_url_params` (dict) can be provided as additional kwargs
                         and will override any other values defined in a configuration
@@ -293,7 +296,7 @@ class AwsDownload(Download):
         )
 
         # authenticate
-        if product.downloader_auth:
+        if product.downloader_auth and isinstance(product.downloader_auth, AwsAuth):
             authenticated_objects = product.downloader_auth.authenticate_objects(
                 bucket_names_and_prefixes
             )
@@ -302,9 +305,19 @@ class AwsDownload(Download):
                 "Authentication plugin (AwsAuth) has to be configured if AwsDownload is used"
             )
 
+        # create an executor if not given and anticipate the possible need to shut it down
+        executor, shutdown_executor = (
+            (ThreadPoolExecutor(), True) if executor is None else (executor, False)
+        )
+        self._config_executor(executor)
+
         # files in zip
         updated_bucket_names_and_prefixes = self._download_file_in_zip(
-            product, bucket_names_and_prefixes, product_local_path, progress_callback
+            product.downloader_auth,
+            bucket_names_and_prefixes,
+            product_local_path,
+            progress_callback,
+            executor,
         )
         # prevent nothing-to-download errors if download was performed in zip
         raise_error = (
@@ -329,7 +342,8 @@ class AwsDownload(Download):
         if len(unique_product_chunks) > 0:
             progress_callback.reset(total=total_size)
         try:
-            for product_chunk in unique_product_chunks:
+
+            def download_chunk(product_chunk: Any) -> None:
                 try:
                     chunk_rel_path = self.get_chunk_dest_path(
                         product,
@@ -339,11 +353,11 @@ class AwsDownload(Download):
                 except NotAvailableError as e:
                     # out of SAFE format chunk
                     logger.warning(e)
-                    continue
+                    return
+
                 chunk_abs_path = os.path.join(product_local_path, chunk_rel_path)
                 chunk_abs_path_dir = os.path.dirname(chunk_abs_path)
-                if not os.path.isdir(chunk_abs_path_dir):
-                    os.makedirs(chunk_abs_path_dir)
+                os.makedirs(chunk_abs_path_dir, exist_ok=True)
 
                 bucket_objects = authenticated_objects.get(product_chunk.bucket_name)
                 extra_args = (
@@ -352,12 +366,31 @@ class AwsDownload(Download):
                     else {}
                 )
                 if not os.path.isfile(chunk_abs_path):
+                    transfer_config = TransferConfig(use_threads=False)
                     product_chunk.Bucket().download_file(
                         product_chunk.key,
                         chunk_abs_path,
                         ExtraArgs=extra_args,
                         Callback=progress_callback,
+                        Config=transfer_config,
                     )
+                return
+
+            # use parallelization if possible.
+            # when products are already downloaded in parallel but the executor has only one worker,
+            # we avoid submitting nested tasks to the executor to prevent deadlocks
+            if (
+                executor._thread_name_prefix == "eodag-download-all"
+                and executor._max_workers == 1
+            ):
+                for product_chunk in unique_product_chunks:
+                    download_chunk(product_chunk)
+            else:
+                futures = (
+                    executor.submit(download_chunk, product_chunk)
+                    for product_chunk in unique_product_chunks
+                )
+                [f.result() for f in as_completed(futures)]
 
         except AuthenticationError as e:
             logger.warning("Unexpected error: %s" % e)
@@ -365,6 +398,9 @@ class AwsDownload(Download):
             raise_if_auth_error(e, self.provider)
             logger.warning("Unexpected error: %s" % e)
 
+        if shutdown_executor:
+            executor.shutdown(wait=True)
+
         # finalize safe product
         if build_safe and product.collection and "S2_MSI" in product.collection:
             self.finalize_s2_safe_product(product_local_path)
@@ -386,31 +422,33 @@ class AwsDownload(Download):
         return product_local_path
 
     def _download_file_in_zip(
-        self, product, bucket_names_and_prefixes, product_local_path, progress_callback
+        self,
+        downloader_auth: AwsAuth,
+        bucket_names_and_prefixes: list[tuple[str, Optional[str]]],
+        product_local_path: str,
+        progress_callback: ProgressCallback,
+        executor: ThreadPoolExecutor,
     ):
         """
         Download file in zip from a prefix like `foo/bar.zip!file.txt`
         """
-        if (
-            not getattr(product, "downloader_auth", None)
-            or product.downloader_auth.s3_resource is None
-        ):
+        if downloader_auth.s3_resource is None:
             logger.debug("Cannot check files in s3 zip without s3 resource")
             return bucket_names_and_prefixes
 
-        s3_client = product.downloader_auth.get_s3_client()
+        s3_client = downloader_auth.get_s3_client()
 
         downloaded = []
-        for i, pack in enumerate(bucket_names_and_prefixes):
+
+        def process_zip_file(i: int, pack: tuple[str, Optional[str]]) -> Optional[int]:
             bucket_name, prefix = pack
-            if ".zip!" in prefix:
+            if prefix is not None and ".zip!" in prefix:
                 splitted_path = prefix.split(".zip!")
                 zip_prefix = f"{splitted_path[0]}.zip"
                 rel_path = splitted_path[-1]
                 dest_file = os.path.join(product_local_path, rel_path)
                 dest_abs_path_dir = os.path.dirname(dest_file)
-                if not os.path.isdir(dest_abs_path_dir):
-                    os.makedirs(dest_abs_path_dir)
+                os.makedirs(dest_abs_path_dir, exist_ok=True)
 
                 zip_file, _ = open_s3_zipped_object(
                     bucket_name, zip_prefix, s3_client, partial=False
@@ -428,7 +466,30 @@ class AwsDownload(Download):
                             output_file.write(zchunk)
                             progress_callback(len(zchunk))
 
-                downloaded.append(i)
+                return i
+            return None
+
+        # use parallelization if possible
+        # when products are already downloaded in parallel but the executor has only one worker,
+        # we avoid submitting nested tasks to the executor to prevent deadlocks
+        if (
+            executor._thread_name_prefix == "eodag-download-all"
+            and executor._max_workers == 1
+        ):
+            for i, pack in enumerate(bucket_names_and_prefixes):
+                result = process_zip_file(i, pack)
+                if result is not None:
+                    downloaded.append(result)
+        else:
+            futures = (
+                executor.submit(process_zip_file, i, pack)
+                for i, pack in enumerate(bucket_names_and_prefixes)
+            )
+
+            for future in as_completed(futures):
+                result = future.result()
+                if result is not None:
+                    downloaded.append(result)
 
         return [
             pack
@@ -710,7 +771,7 @@ class AwsDownload(Download):
             ignore_assets,
             product,
         )
-        if auth and isinstance(auth, boto3.resources.base.ServiceResource):
+        if auth and isinstance(auth, boto3.resource("s3").__class__):
             s3_resource = auth
         else:
             s3_resource = boto3.resource(
@@ -773,6 +834,7 @@ class AwsDownload(Download):
             byte_range,
             compress,
             zip_filename,
+            provider_max_workers=getattr(self.config, "max_workers", None),
         )
 
     def _get_commonpath(
@@ -1112,26 +1174,3 @@ class AwsDownload(Download):
 
         logger.debug(f"Downloading {chunk.key} to {product_path}")
         return product_path
-
-    def download_all(
-        self,
-        products: SearchResult,
-        auth: Optional[Union[AuthBase, S3ServiceResource]] = None,
-        downloaded_callback: Optional[DownloadedCallback] = None,
-        progress_callback: Optional[ProgressCallback] = None,
-        wait: float = DEFAULT_DOWNLOAD_WAIT,
-        timeout: float = DEFAULT_DOWNLOAD_TIMEOUT,
-        **kwargs: Unpack[DownloadConf],
-    ) -> list[str]:
-        """
-        download_all using parent (base plugin) method
-        """
-        return super(AwsDownload, self).download_all(
-            products,
-            auth=auth,
-            downloaded_callback=downloaded_callback,
-            progress_callback=progress_callback,
-            wait=wait,
-            timeout=timeout,
-            **kwargs,
-        )

eodag/plugins/download/base.py

@@ -29,6 +29,8 @@ from pathlib import Path
 from time import sleep
 from typing import TYPE_CHECKING, Any, Callable, Literal, Optional, TypeVar, Union
 
+from concurrent.futures import ThreadPoolExecutor, as_completed
+
 from eodag.api.product.metadata_mapping import ONLINE_STATUS
 from eodag.plugins.base import PluginTopic
 from eodag.utils import (
@@ -105,6 +107,7 @@ class Download(PluginTopic):
         product: EOProduct,
         auth: Optional[Union[AuthBase, S3ServiceResource]] = None,
         progress_callback: Optional[ProgressCallback] = None,
+        executor: Optional[ThreadPoolExecutor] = None,
         wait: float = DEFAULT_DOWNLOAD_WAIT,
         timeout: float = DEFAULT_DOWNLOAD_TIMEOUT,
         **kwargs: Unpack[DownloadConf],
@@ -115,6 +118,7 @@ class Download(PluginTopic):
         :param product: The EO product to download
         :param auth: (optional) authenticated object
         :param progress_callback: (optional) A progress callback
+        :param executor: (optional) An executor to download assets of ``product`` in parallel if it has any
         :param wait: (optional) If download fails, wait time in minutes between two download tries
         :param timeout: (optional) If download fails, maximum time in minutes before stop retrying
                         to download
@@ -447,6 +451,7 @@ class Download(PluginTopic):
         auth: Optional[Union[AuthBase, S3ServiceResource]] = None,
         downloaded_callback: Optional[DownloadedCallback] = None,
         progress_callback: Optional[ProgressCallback] = None,
+        executor: Optional[ThreadPoolExecutor] = None,
         wait: float = DEFAULT_DOWNLOAD_WAIT,
         timeout: float = DEFAULT_DOWNLOAD_TIMEOUT,
         **kwargs: Unpack[DownloadConf],
@@ -454,7 +459,7 @@ class Download(PluginTopic):
         """
         Base download_all method.
 
-        This specific implementation uses the :meth:`eodag.plugins.download.base.Download.download` method
+        This specific implementation uses the :meth:`~eodag.api.product._product.EOProduct.download` method
         implemented by the plugin to **sequentially** attempt to download products.
 
         :param products: Products to download
@@ -465,6 +470,8 @@ class Download(PluginTopic):
                                     its ``__call__`` method. Will be called each time a product
                                     finishes downloading
         :param progress_callback: (optional) A progress callback
+        :param executor: (optional) An executor to download products in parallel which may
+                         be reused to also download assets of these products in parallel.
         :param wait: (optional) If download fails, wait time in minutes between two download tries
         :param timeout: (optional) If download fails, maximum time in minutes before stop retrying
                         to download
@@ -485,9 +492,15 @@ class Download(PluginTopic):
         stop_time = start_time + timedelta(minutes=timeout)
         nb_products = len(products)
         retry_count = 0
-        # another output for notbooks
+        # another output for notebooks
         nb_info = NotebookWidgets()
 
+        # create an executor if not given
+        executor = ThreadPoolExecutor() if executor is None else executor
+        # set thread name prefix so that the EOProduct download() method can identify
+        # whether the executor was created during parallel product downloads or not
+        self._config_executor(executor, "eodag-download-all")
+
         for product in products:
             product.next_try = start_time
 
@@ -508,53 +521,88 @@ class Download(PluginTopic):
             progress_callback.unit_scale = False
         progress_callback.refresh()
 
+        # anticipate nested tasks to download assets in parallel for at least one product
+        nested_asset_downloads = any(
+            product
+            for product in products
+            if (
+                product.downloader
+                and product.downloader.config.type == "AwsDownload"
+                or len(product.assets) > 0
+                and (
+                    not getattr(self.config, "ignore_assets", False)
+                    or kwargs.get("asset") is not None
+                )
+            )
+        )
+
         with progress_callback as bar:
             while "Loop until all products are download or timeout is reached":
-                # try downloading each product before retry
-                for idx, product in enumerate(products):
+                # try downloading each product in parallel before retry
+
+                # Download products in batches to handle nested tasks to download assets in parallel.
+                # We avoid having less workers in the executor than the number of products to download in parallel
+                # to prevent deadlocks. This could happen by submiting and waiting for a task within a task.
+                # We ensure at least one thread is available for these tasks and at least one product is downloaded
+                # at a time.
+                # If there is only one worker, a specific process at assets download level is used to avoid deadlocks.
+                batch_size = len(products)
+                if nested_asset_downloads and executor._max_workers <= batch_size:
+                    batch_size = max(executor._max_workers - 1, 1)
+
+                products_batch = products[:batch_size]
+                futures = {}
+
+                for idx, product in enumerate(products_batch):
                     if datetime.now() >= product.next_try:
                         products[idx].next_try += timedelta(minutes=wait)
-                        try:
-                            paths.append(
-                                product.download(
-                                    progress_callback=product_progress_callback,
-                                    wait=wait,
-                                    timeout=-1,
-                                    **kwargs,
-                                )
-                            )
+                        future = executor.submit(
+                            product.download,
+                            progress_callback=product_progress_callback,
+                            executor=executor,
+                            wait=wait,
+                            timeout=-1,
+                            **kwargs,  # type: ignore
+                        )
+                        futures[future] = product
 
-                            if downloaded_callback:
-                                downloaded_callback(product)
+                for future in as_completed(futures.keys()):
+                    product = futures[future]
+                    try:
+                        result = future.result()
+                        paths.append(result)
 
-                            # product downloaded, to not retry it
-                            products.remove(product)
-                            bar(1)
+                        if downloaded_callback:
+                            downloaded_callback(product)
 
-                            # reset stop time for next product
-                            stop_time = datetime.now() + timedelta(minutes=timeout)
+                        # product downloaded, to not retry it
+                        products.remove(product)
+                        bar(1)
 
-                        except NotAvailableError as e:
-                            logger.info(e)
-                            continue
+                        # reset stop time for next product
+                        stop_time = datetime.now() + timedelta(minutes=timeout)
 
-                        except (AuthenticationError, MisconfiguredError):
-                            logger.exception(
-                                f"Stopped because of credentials problems with provider {self.provider}"
-                            )
-                            raise
+                    except NotAvailableError as e:
+                        logger.info(e)
+                        continue
 
-                        except (RuntimeError, Exception):
-                            import traceback as tb
+                    except (AuthenticationError, MisconfiguredError):
+                        logger.exception(
+                            f"Stopped because of credentials problems with provider {self.provider}"
+                        )
+                        raise
 
-                            logger.error(
-                                f"A problem occurred during download of product: {product}. "
-                                "Skipping it"
-                            )
-                            logger.debug(f"\n{tb.format_exc()}")
+                    except (RuntimeError, Exception):
+                        import traceback as tb
+
+                        logger.error(
+                            f"A problem occurred during download of product: {product}. "
+                            "Skipping it"
+                        )
+                        logger.debug(f"\n{tb.format_exc()}")
 
-                            # product skipped, to not retry it
-                            products.remove(product)
+                        # product skipped, to not retry it
+                        products.remove(product)
 
                 if (
                     len(products) > 0
@@ -567,6 +615,7 @@ class Download(PluginTopic):
                         f"[Retry #{retry_count}, {nb_products - len(products)}/{nb_products} D/L] "
                         f"Waiting {wait_seconds}s until next download try (retry every {wait}' for {timeout}')"
                     )
+
                     logger.info(info_message)
                     nb_info.display_html(info_message)
                     sleep(wait_seconds + 1)
@@ -579,6 +628,9 @@ class Download(PluginTopic):
                 elif len(products) == 0:
                     break
 
+        # Shutdown executor at the end
+        executor.shutdown(wait=True)
+
         return paths
 
     def _order_download_retry(
@@ -641,8 +693,8 @@ class Download(PluginTopic):
                         )
                         logger.info(not_available_info)
                         # Retry-After info from Response header
-                        if hasattr(self, "stream"):
-                            retry_server_info = self.stream.headers.get(
+                        if hasattr(product, "_stream"):
+                            retry_server_info = product._stream.headers.get(
                                 "Retry-After", ""
                             )
                             if retry_server_info:
@@ -663,8 +715,8 @@ class Download(PluginTopic):
                         )
                         logger.info(not_available_info)
                         # Retry-After info from Response header
-                        if hasattr(self, "stream"):
-                            retry_server_info = self.stream.headers.get(
+                        if hasattr(product, "_stream"):
+                            retry_server_info = product._stream.headers.get(
                                 "Retry-After", ""
                             )
                             if retry_server_info:
@@ -690,3 +742,27 @@ class Download(PluginTopic):
             return download_and_retry
 
         return decorator
+
+    def _config_executor(
+        self, executor: ThreadPoolExecutor, thread_name_prefix: Optional[str] = None
+    ) -> None:
+        """
+        Configure a ThreadPoolExecutor instance.
+
+        This method ensures that a ThreadPoolExecutor is correctly set for downloads by adjusting its
+        maximum number of workers if necessary. It also configures the thread name prefix to identify
+        threads created by the executor, which is useful for distinguishing between executors created
+        for parallel product downloads versus those created for other purposes.
+
+        :param executor: A ThreadPoolExecutor instance.
+        :param thread_name_prefix: (optional) A prefix for naming threads created by the executor.
+                                   When provided, threads will be named using this prefix to help
+                                   identify the executor's purpose (e.g., "eodag-download-all").
+        """
+        if (
+            max_workers := getattr(self.config, "max_workers", executor._max_workers)
+        ) < executor._max_workers:
+            executor._max_workers = max_workers
+
+        if thread_name_prefix:
+            executor._thread_name_prefix = "eodag-download-all"

eodag/plugins/download/http.py

@@ -41,6 +41,7 @@ from urllib.parse import parse_qs, urlparse
 
 import geojson
 import requests
+from concurrent.futures import ThreadPoolExecutor, as_completed
 from lxml import etree
 from requests import RequestException
 from requests.auth import AuthBase
@@ -90,10 +91,9 @@ if TYPE_CHECKING:
     from requests import Response
 
     from eodag.api.product import Asset, EOProduct  # type: ignore
-    from eodag.api.search_result import SearchResult
     from eodag.config import PluginConfig
     from eodag.types.download_args import DownloadConf
-    from eodag.utils import DownloadedCallback, Unpack
+    from eodag.utils import Unpack
 
 logger = logging.getLogger("eodag.download.http")
 
@@ -596,6 +596,7 @@ class HTTPDownload(Download):
         product: EOProduct,
         auth: Optional[Union[AuthBase, S3ServiceResource]] = None,
         progress_callback: Optional[ProgressCallback] = None,
+        executor: Optional[ThreadPoolExecutor] = None,
         wait: float = DEFAULT_DOWNLOAD_WAIT,
         timeout: float = DEFAULT_DOWNLOAD_TIMEOUT,
         **kwargs: Unpack[DownloadConf],
@@ -637,6 +638,7 @@ class HTTPDownload(Download):
                     record_filename,
                     auth,
                     progress_callback,
+                    executor,
                     **kwargs,
                 )
                 if kwargs.get("asset") is None:
@@ -674,7 +676,7 @@ class HTTPDownload(Download):
                         is_empty = False
                         progress_callback(len(chunk))
                         fhandle.write(chunk)
-                self.stream.close()  # Closing response stream
+                product._stream.close()  # Closing response stream
 
                 if is_empty:
                     raise DownloadError(f"product {product.properties['id']} is empty")
@@ -720,7 +722,7 @@ class HTTPDownload(Download):
         return product_path
 
     def _check_stream_size(self, product: EOProduct) -> int:
-        stream_size = int(self.stream.headers.get("content-length", 0))
+        stream_size = int(product._stream.headers.get("content-length", 0))
         if (
             stream_size == 0
             and "order:status" in product.properties
@@ -731,14 +733,14 @@ class HTTPDownload(Download):
                 % (
                     product.properties["title"],
                     product.properties["order:status"],
-                    self.stream.reason,
+                    product._stream.reason,
                 )
             )
         return stream_size
 
     def _check_product_filename(self, product: EOProduct) -> str:
         filename = None
-        asset_content_disposition = self.stream.headers.get("content-disposition")
+        asset_content_disposition = product._stream.headers.get("content-disposition")
         if asset_content_disposition:
             filename = cast(
                 Optional[str],
@@ -746,7 +748,7 @@ class HTTPDownload(Download):
             )
         if not filename:
             # default filename extracted from path
-            filename = str(os.path.basename(self.stream.url))
+            filename = str(os.path.basename(product._stream.url))
             filename_extension = os.path.splitext(filename)[1]
             if not filename_extension:
                 if content_type := getattr(product, "headers", {}).get("Content-Type"):
@@ -789,15 +791,20 @@ class HTTPDownload(Download):
             not getattr(self.config, "ignore_assets", False)
             or kwargs.get("asset") is not None
         ):
+            executor = ThreadPoolExecutor(
+                max_workers=getattr(self.config, "max_workers", None)
+            )
             try:
                 assets_values = product.assets.get_values(kwargs.get("asset"))
-                assets_stream_list = self._stream_download_assets(
-                    product,
-                    auth,
-                    None,
-                    assets_values=assets_values,
-                    **kwargs,
-                )
+                with executor:
+                    assets_stream_list = self._stream_download_assets(
+                        product,
+                        executor,
+                        auth,
+                        None,
+                        assets_values,
+                        **kwargs,
+                    )
 
                 # single asset
                 if len(assets_stream_list) == 1:
@@ -1009,7 +1016,7 @@ class HTTPDownload(Download):
 
         s = requests.Session()
         try:
-            self.stream = s.request(
+            product._stream = s.request(
                 req_method,
                 req_url,
                 stream=True,
@@ -1024,7 +1031,7 @@ class HTTPDownload(Download):
             # location is not a valid url -> product is not available yet
             raise NotAvailableError("Product is not available yet")
         try:
-            self.stream.raise_for_status()
+            product._stream.raise_for_status()
         except requests.exceptions.Timeout as exc:
             raise TimeOutError(exc, timeout=DEFAULT_STREAM_REQUESTS_TIMEOUT) from exc
         except RequestException as e:
@@ -1036,8 +1043,8 @@ class HTTPDownload(Download):
             # check if product was ordered
 
             if getattr(
-                self.stream, "status_code", None
-            ) is not None and self.stream.status_code == getattr(
+                product._stream, "status_code", None
+            ) is not None and product._stream.status_code == getattr(
                 self.config, "order_status", {}
             ).get(
                 "ordered", {}
@@ -1048,7 +1055,7 @@ class HTTPDownload(Download):
                 self._process_exception(None, product, ordered_message)
             stream_size = self._check_stream_size(product) or None
 
-            product.headers = self.stream.headers
+            product.headers = product._stream.headers
             filename = self._check_product_filename(product)
             content_type = product.headers.get("Content-Type")
             guessed_content_type = (
@@ -1061,11 +1068,12 @@ class HTTPDownload(Download):
             product.size = stream_size
 
             product.filename = filename
-            return self.stream.iter_content(chunk_size=64 * 1024)
+            return product._stream.iter_content(chunk_size=64 * 1024)
 
     def _stream_download_assets(
         self,
         product: EOProduct,
+        executor: ThreadPoolExecutor,
         auth: Optional[AuthBase] = None,
         progress_callback: Optional[ProgressCallback] = None,
         assets_values: list[Asset] = [],
@@ -1082,7 +1090,9 @@ class HTTPDownload(Download):
             self.config, "dl_url_params", {}
         )
 
-        total_size = self._get_asset_sizes(assets_values, auth, params) or None
+        total_size = (
+            self._get_asset_sizes(assets_values, executor, auth, params) or None
+        )
 
         progress_callback.reset(total=total_size)
 
@@ -1188,11 +1198,6 @@ class HTTPDownload(Download):
 
         # Process each asset
         for asset in assets_values:
-            if not asset["href"] or asset["href"].startswith("file:"):
-                logger.info(
-                    f"Local asset detected. Download skipped for {asset['href']}"
-                )
-                continue
             asset_chunks = get_chunks_generator(asset)
             try:
                 # start reading chunks to set assets attributes
@@ -1220,6 +1225,7 @@ class HTTPDownload(Download):
         record_filename: str,
         auth: Optional[AuthBase] = None,
         progress_callback: Optional[ProgressCallback] = None,
+        executor: Optional[ThreadPoolExecutor] = None,
         **kwargs: Unpack[DownloadConf],
     ) -> str:
         """Download product assets if they exist"""
@@ -1227,6 +1233,12 @@ class HTTPDownload(Download):
             logger.info("Progress bar unavailable, please call product.download()")
             progress_callback = ProgressCallback(disable=True)
 
+        # create an executor if not given and anticipate the possible need to shut it down
+        executor, shutdown_executor = (
+            (ThreadPoolExecutor(), True) if executor is None else (executor, False)
+        )
+        self._config_executor(executor)
+
         assets_urls = [
             a["href"] for a in getattr(product, "assets", {}).values() if "href" in a
         ]
@@ -1236,7 +1248,7 @@ class HTTPDownload(Download):
         assets_values = product.assets.get_values(kwargs.get("asset"))
 
         assets_stream_list = self._stream_download_assets(
-            product, auth, progress_callback, assets_values=assets_values, **kwargs
+            product, executor, auth, progress_callback, assets_values, **kwargs
         )
 
         # remove existing incomplete file
@@ -1259,15 +1271,14 @@ class HTTPDownload(Download):
                 local_assets_count += 1
                 continue
 
-        for asset_stream in assets_stream_list:
+        def download_asset(asset_stream: StreamResponse) -> None:
             asset_chunks = asset_stream.content
             asset_path = cast(str, asset_stream.arcname)
             asset_abs_path = os.path.join(fs_dir_path, asset_path)
             asset_abs_path_temp = asset_abs_path + "~"
             # create asset subdir if not exist
             asset_abs_path_dir = os.path.dirname(asset_abs_path)
-            if not os.path.isdir(asset_abs_path_dir):
-                os.makedirs(asset_abs_path_dir)
+            os.makedirs(asset_abs_path_dir, exist_ok=True)
             # remove temporary file
             if os.path.isfile(asset_abs_path_temp):
                 os.remove(asset_abs_path_temp)
@@ -1283,6 +1294,27 @@ class HTTPDownload(Download):
                     os.path.basename(asset_abs_path),
                 )
                 os.rename(asset_abs_path_temp, asset_abs_path)
+            return
+
+        # use parallelization if possible
+        # when products are already downloaded in parallel but the executor has only one worker,
+        # we avoid submitting nested tasks to the executor to prevent deadlocks
+        if (
+            executor._thread_name_prefix == "eodag-download-all"
+            and executor._max_workers == 1
+        ):
+            for asset_stream in assets_stream_list:
+                download_asset(asset_stream)
+        else:
+            futures = (
+                executor.submit(download_asset, asset_stream)
+                for asset_stream in assets_stream_list
+            )
+            [f.result() for f in as_completed(futures)]
+
+        if shutdown_executor:
+            executor.shutdown(wait=True)
+
         # only one local asset
         if local_assets_count == len(assets_urls) and local_assets_count == 1:
             # remove empty {fs_dir_path}
@@ -1336,6 +1368,7 @@ class HTTPDownload(Download):
     def _get_asset_sizes(
         self,
         assets_values: list[Asset],
+        executor: ThreadPoolExecutor,
         auth: Optional[AuthBase],
         params: Optional[dict[str, str]],
         zipped: bool = False,
@@ -1344,8 +1377,11 @@ class HTTPDownload(Download):
 
         timeout = getattr(self.config, "timeout", HTTP_REQ_TIMEOUT)
         ssl_verify = getattr(self.config, "ssl_verify", True)
-        # loop for assets size & filename
-        for asset in assets_values:
+
+        # loop for assets size & filename in parallel
+        def fetch_asset_size(asset: Asset) -> None:
+            nonlocal total_size
+
             if asset["href"] and not asset["href"].startswith("file:"):
                 # HEAD request for size & filename
                 try:
@@ -1407,27 +1443,20 @@ class HTTPDownload(Download):
                             asset.size = int(size_str) if size_str.isdigit() else 0
 
                 total_size += asset.size
-        return total_size
 
-    def download_all(
-        self,
-        products: SearchResult,
-        auth: Optional[Union[AuthBase, S3ServiceResource]] = None,
-        downloaded_callback: Optional[DownloadedCallback] = None,
-        progress_callback: Optional[ProgressCallback] = None,
-        wait: float = DEFAULT_DOWNLOAD_WAIT,
-        timeout: float = DEFAULT_DOWNLOAD_TIMEOUT,
-        **kwargs: Unpack[DownloadConf],
-    ):
-        """
-        Download all using parent (base plugin) method
-        """
-        return super(HTTPDownload, self).download_all(
-            products,
-            auth=auth,
-            downloaded_callback=downloaded_callback,
-            progress_callback=progress_callback,
-            wait=wait,
-            timeout=timeout,
-            **kwargs,
-        )
+        # use parallelization if possible
+        # when products are already downloaded in parallel but the executor has only one worker,
+        # we avoid submitting nested tasks to the executor to prevent deadlocks
+        if (
+            executor._thread_name_prefix == "eodag-download-all"
+            and executor._max_workers == 1
+        ):
+            for asset in assets_values:
+                fetch_asset_size(asset)
+        else:
+            futures = (
+                executor.submit(fetch_asset_size, asset) for asset in assets_values
+            )
+            [f.result() for f in as_completed(futures)]
+
+        return total_size

eodag/resources/providers.yml

@@ -49,6 +49,7 @@
       usgs:productId: '$.id'
     extract: True
     order_enabled: true
+    max_workers: 2
   products:
     # datasets list http://kapadia.github.io/usgs/_sources/reference/catalog/ee.txt may be outdated
     # see also https://dds.cr.usgs.gov/ee-data/coveragemaps/shp/ee/
@@ -2558,6 +2559,7 @@
     extract: true
     order_enabled: false
     archive_depth: 2
+    max_workers: 4
     ssl_verify: true
   auth: !plugin
     type: KeycloakOIDCPasswordAuth

eodag/utils/s3.py

@@ -391,7 +391,7 @@ def stream_download_from_s3(
     compress: Literal["zip", "raw", "auto"] = "auto",
     zip_filename: str = "archive",
     range_size: int = 1024**2 * 8,
-    max_workers: int = 8,
+    provider_max_workers: Optional[int] = None,
 ) -> StreamResponse:
     """
     Stream data from one or more S3 objects in chunks, with support for global byte ranges.
@@ -431,7 +431,7 @@ def stream_download_from_s3(
         Only used when creating ZIP archives.
     :param range_size: Size of each download chunk in bytes. Larger chunks reduce
         request overhead but use more memory. Default: 8MB.
-    :param max_workers: Maximum number of concurrent download threads.
+    :param provider_max_workers: (optional) Maximum number of concurrent download threads of the provider used.
         Higher values improve throughput for multiple ranges.
     :return: StreamResponse object containing:
 
@@ -480,7 +480,7 @@ def stream_download_from_s3(
             pass
     """
 
-    executor = ThreadPoolExecutor(max_workers=max_workers)
+    executor = ThreadPoolExecutor(max_workers=provider_max_workers)
 
     # Prepare all files
     offset = 0
@@ -596,7 +596,7 @@ def update_assets_from_s3(
 def open_s3_zipped_object(
     bucket_name: str,
     key_name: str,
-    s3_client,
+    s3_client: S3Client,
     zip_size: Optional[int] = None,
     partial: bool = True,
 ) -> tuple[ZipFile, bytes]:

{eodag-4.0.0a4.dist-info → eodag-4.0.0a5.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: eodag
-Version: 4.0.0a4
+Version: 4.0.0a5
 Summary: Earth Observation Data Access Gateway
 Home-page: https://github.com/CS-SI/eodag
 Author: CS GROUP - France

{eodag-4.0.0a4.dist-info → eodag-4.0.0a5.dist-info}/RECORD

@@ -1,16 +1,16 @@
 eodag/__init__.py,sha256=vS9nq3ZOcdlHxCakeRqCV4nm8q3r7ZKr-p7RQg0gatg,1627
-eodag/cli.py,sha256=zTb9oyq8BBJI1Iftfp_8iJF_EHaAPr_c_4SyLnyMcaA,22023
+eodag/cli.py,sha256=QZqnhopBdPasNWdlti7n2VQ8sYylUoWIKT7_P1ev-pw,22340
 eodag/config.py,sha256=ZdWE4rZCcR8OX5jGv7zKEQ4CgH5bNk_PpNGe8HlhjR4,35054
 eodag/crunch.py,sha256=fLVAPGVPw31N_DrnFk4gkCpQZLMY8oBhK6NUSYmdr24,1099
 eodag/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 eodag/api/__init__.py,sha256=ytr30NUVmEtmJTsp3QCwkCIhS1nF6UlFCv0vmySHN7g,735
 eodag/api/collection.py,sha256=ZEKTB4yKJAIQfjAFkhHZ76Vw-KQ9qPAmpzkZqJ4KGUU,13804
-eodag/api/core.py,sha256=c0RWQJgVluTQCopVHZMmiqhusUdZcekDQZILiyyaMmc,102269
+eodag/api/core.py,sha256=KJWj_HRmZpMfrjvzMgLVLO4SwXrirGJfPp9rRHIo6J4,103037
 eodag/api/provider.py,sha256=ZvkmJNYRCcYMP18X5KLSVhXZTvG1gkltfah1OA4LV-8,37279
 eodag/api/search_result.py,sha256=LGWIQgbhQF5Lz489vcvalb2VsYtplyzUXGWMqduP0uc,21107
 eodag/api/product/__init__.py,sha256=6-RXKAw-sZVNU-7KjpmSqLc35dfc0V42wI5GqIzH2QA,2501
 eodag/api/product/_assets.py,sha256=9bWIe_SYvsQO-q_lQFd7SxhUIWv7feze2-wnXOe8hhs,7570
-eodag/api/product/_product.py,sha256=Sxc1VnNWbtivpAdpwxmTjLCFQ8P7JedzFnIRZA25hO4,27475
+eodag/api/product/_product.py,sha256=CdoQG7SuX-lNfd4Ob3M9ET7jdNSovEekS5_Jq0AmIao,28456
 eodag/api/product/metadata_mapping.py,sha256=OhB5JhyMLwCrXYJbdk2Nep0Kd-lFeiCWLtw1pa3XukU,76821
 eodag/api/product/drivers/__init__.py,sha256=v85lXHsfrkn59uyMgConb5ZY9acCCx95srdsh_wFIlM,1998
 eodag/api/product/drivers/base.py,sha256=MvaXicICn9DxXsaJuK34nPM3__TksstZXSVfABcINAc,2776
@@ -22,8 +22,8 @@ eodag/plugins/base.py,sha256=8Us6InkQu59Ya0Ne8se2Q6ZtU85IpCpcQqO9Z2ECDWI,2746
 eodag/plugins/manager.py,sha256=4annSLfT6N3o-japomvIUCTpDXfMgzfhBkvHVSKbVf0,19748
 eodag/plugins/apis/__init__.py,sha256=PyY4f7P2iu3MkLPnw5eOrVew2fuavbBL3Asci3Ulwoo,744
 eodag/plugins/apis/base.py,sha256=PZwAg0uINSEQcHSzTgU9xSAb-WaGMbmDL-jhrmybtLQ,2814
-eodag/plugins/apis/ecmwf.py,sha256=6_Zr3u2XHJl6XijNZOWazdtqi8v_b9D4WNET-Eub5qs,11767
-eodag/plugins/apis/usgs.py,sha256=V1R13tqPgVJ_kMoEdbbFT-lx5mZ7yYqRtna--wmTXAk,20029
+eodag/plugins/apis/ecmwf.py,sha256=qjELC_rwSY3Q3aZbwmJ8-eBq9FVZDt4uWdjK77eB-34,11075
+eodag/plugins/apis/usgs.py,sha256=e8QFXg3zqFyMx6R3-gChJzWjUNZ65VULLsrOTqKXwLM,19338
 eodag/plugins/authentication/__init__.py,sha256=_LVw42Bb1IhGAZH5xHRaS4b1iFoF9e27KDZOyoSoJHY,1039
 eodag/plugins/authentication/aws_auth.py,sha256=A0sr5mVmCkAwCldlGSGFcZqnthusbcB52THthja77cw,12651
 eodag/plugins/authentication/base.py,sha256=wp-OI0G4DbUFykuehvyh4IJLAJyicyKiIH9k5z3W7Mo,3506
@@ -43,9 +43,9 @@ eodag/plugins/crunch/filter_latest_tpl_name.py,sha256=Aau0RMBRCblVL2kapEaV0SMZXd
 eodag/plugins/crunch/filter_overlap.py,sha256=tU3h15jyfePOQPgGvowHFm2ZuckEbAo6lgLQNaPiJ2Q,7142
 eodag/plugins/crunch/filter_property.py,sha256=2BKb7wxw1Yi2NTtnPCBtdZ-caJXxlVUUS2ps4LHXOMI,3187
 eodag/plugins/download/__init__.py,sha256=zqszaeNgYP0YHlZDkLMf6odcwNw0KrAahGpcA-l0kAw,740
-eodag/plugins/download/aws.py,sha256=x0HWR9O5U3kFo7W8qHcZhkCqzWEkSw_XEVIdx1zX7Qg,46884
-eodag/plugins/download/base.py,sha256=TVUCDMuDkRo9EKYj5o3zUuOl76VJfQ6GlefD1EFwSTs,30807
-eodag/plugins/download/http.py,sha256=DDeHZkVi6E0Zi7YQxRB9z8AENKwBsUdri68mozVI5OI,58608
+eodag/plugins/download/aws.py,sha256=n3C3XZl-wFdCzsSB-OuDem985iFPVCnfAy4SWUjOkj8,48653
+eodag/plugins/download/base.py,sha256=--30euISg80V_STkwRUhg6ocvQnQuSWJwlnh5FMv8ts,34504
+eodag/plugins/download/http.py,sha256=DB46ff8UvXXOykrGef3qXWar5NtXqWjdswdlYvf3Nwk,59792
 eodag/plugins/search/__init__.py,sha256=z_OD0bIloltQIJ9D0-pLC6o6nT0VmX2PRtMn_nLwWDQ,2174
 eodag/plugins/search/base.py,sha256=yBmfca1EcR8c8pDD0GLnyZkU1UQ_8RNh1X8xqV-gHRQ,22235
 eodag/plugins/search/build_search_result.py,sha256=4QZKXEBj5n1EE0BGciLfApSEYxxk-2srlVrq27lL1hg,62934
@@ -59,7 +59,7 @@ eodag/resources/collections.yml,sha256=0JZr5y8V8QlB1MGqGzs5-Pmrk1ZYkEBlS3VYrg22N
 eodag/resources/ext_collections.json,sha256=_HLFBMx6jSaFG2ZYAbTqSZUr9Rf87tQgLd8Ibad4khc,2587896
 eodag/resources/ext_product_types.json,sha256=uNhgmppM6ZrxPSWWfR1rANavy1f88stFORElk-WkCxA,2607329
 eodag/resources/locations_conf_template.yml,sha256=_eBv-QKHYMIKhY0b0kp4Ee33RsayxN8LWH3kDXxfFSk,986
-eodag/resources/providers.yml,sha256=2ryHvNIaBpETbAJUQzXAbY08crWxzbBUfTJ1yprVZr0,237882
+eodag/resources/providers.yml,sha256=yBz5S-E5MR_zdp5yTGbkQ6I9Tk9Kax_Pdk0AstggUqM,237920
 eodag/resources/stac_provider.yml,sha256=RNhFNhv7593VkY2od-LQgUBdIO1WXL1Rko7UyjLaxFY,4648
 eodag/resources/user_conf_template.yml,sha256=aHSiscSQ3B4Dd18709iQAX7tFrkMufRC1a9akcNVVTs,7541
 eodag/resources/shp/ne_110m_admin_0_map_units.VERSION.txt,sha256=CHSo_jbv-4d4D0MYRbWn2FvmV_K9mYzo7qznF4YNO3g,7
@@ -84,11 +84,11 @@ eodag/utils/logging.py,sha256=KoMsyS1f6O1hr_SMDOIxvt842mOJgmu_yLUk0-0EKFs,3507
 eodag/utils/notebook.py,sha256=AUxtayvu26qYf3x3Eu3ujRl1XDgy24EfQaETbqmXSZw,2703
 eodag/utils/repr.py,sha256=72BIKFq07aU4YrQVJJX-AADdWXAhJqC4LXGmkbCo1kA,5537
 eodag/utils/requests.py,sha256=avNHKrOZ7Kp6lUA7u4kqupIth9MoirLzDsMrrmQDt4s,4560
-eodag/utils/s3.py,sha256=eESanPLVv-Luqo_o1WgUuO7YLqiXg_iEzHZ15fu-ugY,30063
+eodag/utils/s3.py,sha256=Yu-yF0ebIm03XOm94p4SkDi2WzU_ZTv4kVJ3VGBo-Z4,30145
 eodag/utils/stac_reader.py,sha256=8r6amio5EtwGF9iu9zHaGDz4oUPKKeXRuyTzPNakrO4,9406
-eodag-4.0.0a4.dist-info/licenses/LICENSE,sha256=4MAecetnRTQw5DlHtiikDSzKWO1xVLwzM5_DsPMYlnE,10172
-eodag-4.0.0a4.dist-info/METADATA,sha256=deJg_rjOM8_P2iTCZWGswMBYbVZmrSZDkAGxRlgn8cA,12739
-eodag-4.0.0a4.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
-eodag-4.0.0a4.dist-info/entry_points.txt,sha256=atMIh-Q4hRsOdw1_778mDIhWFHQJigEo3x-0fMqhqLE,2254
-eodag-4.0.0a4.dist-info/top_level.txt,sha256=025IMTmVe5eDjIPP4KEFQKespOPMQdne4U4jOy8nftM,6
-eodag-4.0.0a4.dist-info/RECORD,,
+eodag-4.0.0a5.dist-info/licenses/LICENSE,sha256=4MAecetnRTQw5DlHtiikDSzKWO1xVLwzM5_DsPMYlnE,10172
+eodag-4.0.0a5.dist-info/METADATA,sha256=NccWfNN7f57un7W4xCpEBhpt2UWQFYSjNImlVg9uvx8,12739
+eodag-4.0.0a5.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+eodag-4.0.0a5.dist-info/entry_points.txt,sha256=atMIh-Q4hRsOdw1_778mDIhWFHQJigEo3x-0fMqhqLE,2254
+eodag-4.0.0a5.dist-info/top_level.txt,sha256=025IMTmVe5eDjIPP4KEFQKespOPMQdne4U4jOy8nftM,6
+eodag-4.0.0a5.dist-info/RECORD,,