eodag 4.0.0a3__py3-none-any.whl → 4.0.0a5__py3-none-any.whl

eodag/plugins/apis/ecmwf.py

@@ -48,6 +48,7 @@ from eodag.utils.logging import get_logging_verbose
 if TYPE_CHECKING:
     from typing import Any, Optional, Union
 
+    from concurrent.futures import ThreadPoolExecutor
     from mypy_boto3_s3 import S3ServiceResource
     from requests.auth import AuthBase
 
@@ -55,7 +56,7 @@ if TYPE_CHECKING:
     from eodag.api.search_result import SearchResult
     from eodag.config import PluginConfig
     from eodag.types.download_args import DownloadConf
-    from eodag.utils import DownloadedCallback, ProgressCallback, Unpack
+    from eodag.utils import ProgressCallback, Unpack
 
 
 logger = logging.getLogger("eodag.apis.ecmwf")
@@ -185,6 +186,7 @@ class EcmwfApi(Api, ECMWFSearch):
         product: EOProduct,
         auth: Optional[Union[AuthBase, S3ServiceResource]] = None,
         progress_callback: Optional[ProgressCallback] = None,
+        executor: Optional[ThreadPoolExecutor] = None,
         wait: float = DEFAULT_DOWNLOAD_WAIT,
         timeout: float = DEFAULT_DOWNLOAD_TIMEOUT,
         **kwargs: Unpack[DownloadConf],
@@ -269,29 +271,6 @@ class EcmwfApi(Api, ECMWFSearch):
         product.location = path_to_uri(product_path)
         return product_path
 
-    def download_all(
-        self,
-        products: SearchResult,
-        auth: Optional[Union[AuthBase, S3ServiceResource]] = None,
-        downloaded_callback: Optional[DownloadedCallback] = None,
-        progress_callback: Optional[ProgressCallback] = None,
-        wait: float = DEFAULT_DOWNLOAD_WAIT,
-        timeout: float = DEFAULT_DOWNLOAD_TIMEOUT,
-        **kwargs: Unpack[DownloadConf],
-    ) -> list[str]:
-        """
-        Download all using parent (base plugin) method
-        """
-        return super(EcmwfApi, self).download_all(
-            products,
-            auth=auth,
-            downloaded_callback=downloaded_callback,
-            progress_callback=progress_callback,
-            wait=wait,
-            timeout=timeout,
-            **kwargs,
-        )
-
     def clear(self) -> None:
         """Clear search context"""
         pass

eodag/plugins/apis/usgs.py

@@ -57,12 +57,13 @@ from eodag.utils.exceptions import (
 )
 
 if TYPE_CHECKING:
+    from concurrent.futures import ThreadPoolExecutor
     from mypy_boto3_s3 import S3ServiceResource
     from requests.auth import AuthBase
 
     from eodag.config import PluginConfig
     from eodag.types.download_args import DownloadConf
-    from eodag.utils import DownloadedCallback, Unpack
+    from eodag.utils import Unpack
 
 logger = logging.getLogger("eodag.apis.usgs")
 
@@ -312,6 +313,7 @@ class UsgsApi(Api):
         product: EOProduct,
         auth: Optional[Union[AuthBase, S3ServiceResource]] = None,
         progress_callback: Optional[ProgressCallback] = None,
+        executor: Optional[ThreadPoolExecutor] = None,
         wait: float = DEFAULT_DOWNLOAD_WAIT,
         timeout: float = DEFAULT_DOWNLOAD_TIMEOUT,
         **kwargs: Unpack[DownloadConf],
@@ -477,26 +479,3 @@ class UsgsApi(Api):
             shutil.move(fs_path, new_fs_path)
             product.location = path_to_uri(new_fs_path)
             return new_fs_path
-
-    def download_all(
-        self,
-        products: SearchResult,
-        auth: Optional[Union[AuthBase, S3ServiceResource]] = None,
-        downloaded_callback: Optional[DownloadedCallback] = None,
-        progress_callback: Optional[ProgressCallback] = None,
-        wait: float = DEFAULT_DOWNLOAD_WAIT,
-        timeout: float = DEFAULT_DOWNLOAD_TIMEOUT,
-        **kwargs: Unpack[DownloadConf],
-    ) -> list[str]:
-        """
-        Download all using parent (base plugin) method
-        """
-        return super(UsgsApi, self).download_all(
-            products,
-            auth=auth,
-            downloaded_callback=downloaded_callback,
-            progress_callback=progress_callback,
-            wait=wait,
-            timeout=timeout,
-            **kwargs,
-        )

eodag/plugins/authentication/token.py

@@ -226,7 +226,6 @@ class TokenAuth(Authentication):
         self,
         session: requests.Session,
     ) -> requests.Response:
-
         retry_total = getattr(self.config, "retry_total", REQ_RETRY_TOTAL)
         retry_backoff_factor = getattr(
             self.config, "retry_backoff_factor", REQ_RETRY_BACKOFF_FACTOR

eodag/plugins/download/aws.py

@@ -25,7 +25,9 @@ from typing import TYPE_CHECKING, Any, Literal, Optional, Union, cast
 
 import boto3
 import requests
+from boto3.s3.transfer import TransferConfig
 from botocore.exceptions import ClientError
+from concurrent.futures import ThreadPoolExecutor, as_completed
 from lxml import etree
 from requests.auth import AuthBase
 
@@ -34,7 +36,7 @@ from eodag.api.product.metadata_mapping import (
     properties_from_json,
     properties_from_xml,
 )
-from eodag.plugins.authentication.aws_auth import raise_if_auth_error
+from eodag.plugins.authentication.aws_auth import AwsAuth, raise_if_auth_error
 from eodag.plugins.download.base import Download
 from eodag.utils import (
     DEFAULT_DOWNLOAD_TIMEOUT,
@@ -65,10 +67,9 @@ if TYPE_CHECKING:
     from mypy_boto3_s3.client import S3Client
 
     from eodag.api.product import EOProduct
-    from eodag.api.search_result import SearchResult
     from eodag.config import PluginConfig
     from eodag.types.download_args import DownloadConf
-    from eodag.utils import DownloadedCallback, Unpack
+    from eodag.utils import Unpack
 
 
 logger = logging.getLogger("eodag.download.aws")
@@ -227,6 +228,7 @@ class AwsDownload(Download):
         product: EOProduct,
         auth: Optional[Union[AuthBase, S3ServiceResource]] = None,
         progress_callback: Optional[ProgressCallback] = None,
+        executor: Optional[ThreadPoolExecutor] = None,
         wait: float = DEFAULT_DOWNLOAD_WAIT,
         timeout: float = DEFAULT_DOWNLOAD_TIMEOUT,
         **kwargs: Unpack[DownloadConf],
@@ -246,6 +248,7 @@ class AwsDownload(Download):
                                   size as inputs and handle progress bar
                                   creation and update to give the user a
                                   feedback on the download progress
+        :param executor: (optional) An executor to download assets of ``product`` in parallel if it has any
         :param kwargs: `output_dir` (str), `extract` (bool), `delete_archive` (bool)
                         and `dl_url_params` (dict) can be provided as additional kwargs
                         and will override any other values defined in a configuration
@@ -293,7 +296,7 @@ class AwsDownload(Download):
         )
 
         # authenticate
-        if product.downloader_auth:
+        if product.downloader_auth and isinstance(product.downloader_auth, AwsAuth):
             authenticated_objects = product.downloader_auth.authenticate_objects(
                 bucket_names_and_prefixes
             )
@@ -302,9 +305,19 @@ class AwsDownload(Download):
                 "Authentication plugin (AwsAuth) has to be configured if AwsDownload is used"
             )
 
+        # create an executor if not given and anticipate the possible need to shut it down
+        executor, shutdown_executor = (
+            (ThreadPoolExecutor(), True) if executor is None else (executor, False)
+        )
+        self._config_executor(executor)
+
         # files in zip
         updated_bucket_names_and_prefixes = self._download_file_in_zip(
-            product, bucket_names_and_prefixes, product_local_path, progress_callback
+            product.downloader_auth,
+            bucket_names_and_prefixes,
+            product_local_path,
+            progress_callback,
+            executor,
         )
         # prevent nothing-to-download errors if download was performed in zip
         raise_error = (
@@ -329,7 +342,8 @@ class AwsDownload(Download):
         if len(unique_product_chunks) > 0:
             progress_callback.reset(total=total_size)
         try:
-            for product_chunk in unique_product_chunks:
+
+            def download_chunk(product_chunk: Any) -> None:
                 try:
                     chunk_rel_path = self.get_chunk_dest_path(
                         product,
@@ -339,11 +353,11 @@ class AwsDownload(Download):
                 except NotAvailableError as e:
                     # out of SAFE format chunk
                     logger.warning(e)
-                    continue
+                    return
+
                 chunk_abs_path = os.path.join(product_local_path, chunk_rel_path)
                 chunk_abs_path_dir = os.path.dirname(chunk_abs_path)
-                if not os.path.isdir(chunk_abs_path_dir):
-                    os.makedirs(chunk_abs_path_dir)
+                os.makedirs(chunk_abs_path_dir, exist_ok=True)
 
                 bucket_objects = authenticated_objects.get(product_chunk.bucket_name)
                 extra_args = (
@@ -352,12 +366,31 @@ class AwsDownload(Download):
                     else {}
                 )
                 if not os.path.isfile(chunk_abs_path):
+                    transfer_config = TransferConfig(use_threads=False)
                     product_chunk.Bucket().download_file(
                         product_chunk.key,
                         chunk_abs_path,
                         ExtraArgs=extra_args,
                         Callback=progress_callback,
+                        Config=transfer_config,
                     )
+                return
+
+            # use parallelization if possible.
+            # when products are already downloaded in parallel but the executor has only one worker,
+            # we avoid submitting nested tasks to the executor to prevent deadlocks
+            if (
+                executor._thread_name_prefix == "eodag-download-all"
+                and executor._max_workers == 1
+            ):
+                for product_chunk in unique_product_chunks:
+                    download_chunk(product_chunk)
+            else:
+                futures = (
+                    executor.submit(download_chunk, product_chunk)
+                    for product_chunk in unique_product_chunks
+                )
+                [f.result() for f in as_completed(futures)]
 
         except AuthenticationError as e:
             logger.warning("Unexpected error: %s" % e)
@@ -365,6 +398,9 @@ class AwsDownload(Download):
             raise_if_auth_error(e, self.provider)
             logger.warning("Unexpected error: %s" % e)
 
+        if shutdown_executor:
+            executor.shutdown(wait=True)
+
         # finalize safe product
         if build_safe and product.collection and "S2_MSI" in product.collection:
             self.finalize_s2_safe_product(product_local_path)
@@ -386,31 +422,33 @@ class AwsDownload(Download):
         return product_local_path
 
     def _download_file_in_zip(
-        self, product, bucket_names_and_prefixes, product_local_path, progress_callback
+        self,
+        downloader_auth: AwsAuth,
+        bucket_names_and_prefixes: list[tuple[str, Optional[str]]],
+        product_local_path: str,
+        progress_callback: ProgressCallback,
+        executor: ThreadPoolExecutor,
     ):
         """
         Download file in zip from a prefix like `foo/bar.zip!file.txt`
         """
-        if (
-            not getattr(product, "downloader_auth", None)
-            or product.downloader_auth.s3_resource is None
-        ):
+        if downloader_auth.s3_resource is None:
             logger.debug("Cannot check files in s3 zip without s3 resource")
             return bucket_names_and_prefixes
 
-        s3_client = product.downloader_auth.get_s3_client()
+        s3_client = downloader_auth.get_s3_client()
 
         downloaded = []
-        for i, pack in enumerate(bucket_names_and_prefixes):
+
+        def process_zip_file(i: int, pack: tuple[str, Optional[str]]) -> Optional[int]:
             bucket_name, prefix = pack
-            if ".zip!" in prefix:
+            if prefix is not None and ".zip!" in prefix:
                 splitted_path = prefix.split(".zip!")
                 zip_prefix = f"{splitted_path[0]}.zip"
                 rel_path = splitted_path[-1]
                 dest_file = os.path.join(product_local_path, rel_path)
                 dest_abs_path_dir = os.path.dirname(dest_file)
-                if not os.path.isdir(dest_abs_path_dir):
-                    os.makedirs(dest_abs_path_dir)
+                os.makedirs(dest_abs_path_dir, exist_ok=True)
 
                 zip_file, _ = open_s3_zipped_object(
                     bucket_name, zip_prefix, s3_client, partial=False
@@ -428,7 +466,30 @@ class AwsDownload(Download):
                             output_file.write(zchunk)
                             progress_callback(len(zchunk))
 
-                downloaded.append(i)
+                return i
+            return None
+
+        # use parallelization if possible
+        # when products are already downloaded in parallel but the executor has only one worker,
+        # we avoid submitting nested tasks to the executor to prevent deadlocks
+        if (
+            executor._thread_name_prefix == "eodag-download-all"
+            and executor._max_workers == 1
+        ):
+            for i, pack in enumerate(bucket_names_and_prefixes):
+                result = process_zip_file(i, pack)
+                if result is not None:
+                    downloaded.append(result)
+        else:
+            futures = (
+                executor.submit(process_zip_file, i, pack)
+                for i, pack in enumerate(bucket_names_and_prefixes)
+            )
+
+            for future in as_completed(futures):
+                result = future.result()
+                if result is not None:
+                    downloaded.append(result)
 
         return [
             pack
@@ -710,7 +771,7 @@ class AwsDownload(Download):
             ignore_assets,
             product,
         )
-        if auth and isinstance(auth, boto3.resources.base.ServiceResource):
+        if auth and isinstance(auth, boto3.resource("s3").__class__):
             s3_resource = auth
         else:
             s3_resource = boto3.resource(
@@ -773,6 +834,7 @@ class AwsDownload(Download):
             byte_range,
             compress,
             zip_filename,
+            provider_max_workers=getattr(self.config, "max_workers", None),
         )
 
     def _get_commonpath(
@@ -1112,26 +1174,3 @@ class AwsDownload(Download):
 
         logger.debug(f"Downloading {chunk.key} to {product_path}")
         return product_path
-
-    def download_all(
-        self,
-        products: SearchResult,
-        auth: Optional[Union[AuthBase, S3ServiceResource]] = None,
-        downloaded_callback: Optional[DownloadedCallback] = None,
-        progress_callback: Optional[ProgressCallback] = None,
-        wait: float = DEFAULT_DOWNLOAD_WAIT,
-        timeout: float = DEFAULT_DOWNLOAD_TIMEOUT,
-        **kwargs: Unpack[DownloadConf],
-    ) -> list[str]:
-        """
-        download_all using parent (base plugin) method
-        """
-        return super(AwsDownload, self).download_all(
-            products,
-            auth=auth,
-            downloaded_callback=downloaded_callback,
-            progress_callback=progress_callback,
-            wait=wait,
-            timeout=timeout,
-            **kwargs,
-        )

eodag/plugins/download/base.py

@@ -29,6 +29,8 @@ from pathlib import Path
 from time import sleep
 from typing import TYPE_CHECKING, Any, Callable, Literal, Optional, TypeVar, Union
 
+from concurrent.futures import ThreadPoolExecutor, as_completed
+
 from eodag.api.product.metadata_mapping import ONLINE_STATUS
 from eodag.plugins.base import PluginTopic
 from eodag.utils import (
@@ -105,6 +107,7 @@ class Download(PluginTopic):
         product: EOProduct,
         auth: Optional[Union[AuthBase, S3ServiceResource]] = None,
         progress_callback: Optional[ProgressCallback] = None,
+        executor: Optional[ThreadPoolExecutor] = None,
         wait: float = DEFAULT_DOWNLOAD_WAIT,
         timeout: float = DEFAULT_DOWNLOAD_TIMEOUT,
         **kwargs: Unpack[DownloadConf],
@@ -115,6 +118,7 @@ class Download(PluginTopic):
         :param product: The EO product to download
         :param auth: (optional) authenticated object
         :param progress_callback: (optional) A progress callback
+        :param executor: (optional) An executor to download assets of ``product`` in parallel if it has any
         :param wait: (optional) If download fails, wait time in minutes between two download tries
         :param timeout: (optional) If download fails, maximum time in minutes before stop retrying
                         to download
@@ -447,6 +451,7 @@ class Download(PluginTopic):
         auth: Optional[Union[AuthBase, S3ServiceResource]] = None,
         downloaded_callback: Optional[DownloadedCallback] = None,
         progress_callback: Optional[ProgressCallback] = None,
+        executor: Optional[ThreadPoolExecutor] = None,
         wait: float = DEFAULT_DOWNLOAD_WAIT,
         timeout: float = DEFAULT_DOWNLOAD_TIMEOUT,
         **kwargs: Unpack[DownloadConf],
@@ -454,7 +459,7 @@ class Download(PluginTopic):
         """
         Base download_all method.
 
-        This specific implementation uses the :meth:`eodag.plugins.download.base.Download.download` method
+        This specific implementation uses the :meth:`~eodag.api.product._product.EOProduct.download` method
         implemented by the plugin to **sequentially** attempt to download products.
 
         :param products: Products to download
@@ -465,6 +470,8 @@ class Download(PluginTopic):
                                     its ``__call__`` method. Will be called each time a product
                                     finishes downloading
         :param progress_callback: (optional) A progress callback
+        :param executor: (optional) An executor to download products in parallel which may
+                         be reused to also download assets of these products in parallel.
         :param wait: (optional) If download fails, wait time in minutes between two download tries
         :param timeout: (optional) If download fails, maximum time in minutes before stop retrying
                         to download
@@ -485,9 +492,15 @@ class Download(PluginTopic):
         stop_time = start_time + timedelta(minutes=timeout)
         nb_products = len(products)
         retry_count = 0
-        # another output for notbooks
+        # another output for notebooks
         nb_info = NotebookWidgets()
 
+        # create an executor if not given
+        executor = ThreadPoolExecutor() if executor is None else executor
+        # set thread name prefix so that the EOProduct download() method can identify
+        # whether the executor was created during parallel product downloads or not
+        self._config_executor(executor, "eodag-download-all")
+
         for product in products:
             product.next_try = start_time
 
@@ -508,53 +521,88 @@ class Download(PluginTopic):
             progress_callback.unit_scale = False
         progress_callback.refresh()
 
+        # anticipate nested tasks to download assets in parallel for at least one product
+        nested_asset_downloads = any(
+            product
+            for product in products
+            if (
+                product.downloader
+                and product.downloader.config.type == "AwsDownload"
+                or len(product.assets) > 0
+                and (
+                    not getattr(self.config, "ignore_assets", False)
+                    or kwargs.get("asset") is not None
+                )
+            )
+        )
+
         with progress_callback as bar:
             while "Loop until all products are download or timeout is reached":
-                # try downloading each product before retry
-                for idx, product in enumerate(products):
+                # try downloading each product in parallel before retry
+
+                # Download products in batches to handle nested tasks to download assets in parallel.
+                # We avoid having less workers in the executor than the number of products to download in parallel
+                # to prevent deadlocks. This could happen by submiting and waiting for a task within a task.
+                # We ensure at least one thread is available for these tasks and at least one product is downloaded
+                # at a time.
+                # If there is only one worker, a specific process at assets download level is used to avoid deadlocks.
+                batch_size = len(products)
+                if nested_asset_downloads and executor._max_workers <= batch_size:
+                    batch_size = max(executor._max_workers - 1, 1)
+
+                products_batch = products[:batch_size]
+                futures = {}
+
+                for idx, product in enumerate(products_batch):
                     if datetime.now() >= product.next_try:
                         products[idx].next_try += timedelta(minutes=wait)
-                        try:
-                            paths.append(
-                                product.download(
-                                    progress_callback=product_progress_callback,
-                                    wait=wait,
-                                    timeout=-1,
-                                    **kwargs,
-                                )
-                            )
+                        future = executor.submit(
+                            product.download,
+                            progress_callback=product_progress_callback,
+                            executor=executor,
+                            wait=wait,
+                            timeout=-1,
+                            **kwargs,  # type: ignore
+                        )
+                        futures[future] = product
 
-                            if downloaded_callback:
-                                downloaded_callback(product)
+                for future in as_completed(futures.keys()):
+                    product = futures[future]
+                    try:
+                        result = future.result()
+                        paths.append(result)
 
-                            # product downloaded, to not retry it
-                            products.remove(product)
-                            bar(1)
+                        if downloaded_callback:
+                            downloaded_callback(product)
 
-                            # reset stop time for next product
-                            stop_time = datetime.now() + timedelta(minutes=timeout)
+                        # product downloaded, to not retry it
+                        products.remove(product)
+                        bar(1)
 
-                        except NotAvailableError as e:
-                            logger.info(e)
-                            continue
+                        # reset stop time for next product
+                        stop_time = datetime.now() + timedelta(minutes=timeout)
 
-                        except (AuthenticationError, MisconfiguredError):
-                            logger.exception(
-                                f"Stopped because of credentials problems with provider {self.provider}"
-                            )
-                            raise
+                    except NotAvailableError as e:
+                        logger.info(e)
+                        continue
 
-                        except (RuntimeError, Exception):
-                            import traceback as tb
+                    except (AuthenticationError, MisconfiguredError):
+                        logger.exception(
+                            f"Stopped because of credentials problems with provider {self.provider}"
+                        )
+                        raise
 
-                            logger.error(
-                                f"A problem occurred during download of product: {product}. "
-                                "Skipping it"
-                            )
-                            logger.debug(f"\n{tb.format_exc()}")
+                    except (RuntimeError, Exception):
+                        import traceback as tb
+
+                        logger.error(
+                            f"A problem occurred during download of product: {product}. "
+                            "Skipping it"
+                        )
+                        logger.debug(f"\n{tb.format_exc()}")
 
-                            # product skipped, to not retry it
-                            products.remove(product)
+                        # product skipped, to not retry it
+                        products.remove(product)
 
                 if (
                     len(products) > 0
@@ -567,6 +615,7 @@ class Download(PluginTopic):
                         f"[Retry #{retry_count}, {nb_products - len(products)}/{nb_products} D/L] "
                         f"Waiting {wait_seconds}s until next download try (retry every {wait}' for {timeout}')"
                     )
+
                     logger.info(info_message)
                     nb_info.display_html(info_message)
                     sleep(wait_seconds + 1)
@@ -579,6 +628,9 @@ class Download(PluginTopic):
                 elif len(products) == 0:
                     break
 
+        # Shutdown executor at the end
+        executor.shutdown(wait=True)
+
         return paths
 
     def _order_download_retry(
@@ -641,8 +693,8 @@ class Download(PluginTopic):
                         )
                         logger.info(not_available_info)
                         # Retry-After info from Response header
-                        if hasattr(self, "stream"):
-                            retry_server_info = self.stream.headers.get(
+                        if hasattr(product, "_stream"):
+                            retry_server_info = product._stream.headers.get(
                                 "Retry-After", ""
                             )
                             if retry_server_info:
@@ -663,8 +715,8 @@ class Download(PluginTopic):
                         )
                         logger.info(not_available_info)
                         # Retry-After info from Response header
-                        if hasattr(self, "stream"):
-                            retry_server_info = self.stream.headers.get(
+                        if hasattr(product, "_stream"):
+                            retry_server_info = product._stream.headers.get(
                                 "Retry-After", ""
                             )
                             if retry_server_info:
@@ -690,3 +742,27 @@ class Download(PluginTopic):
             return download_and_retry
 
         return decorator
+
+    def _config_executor(
+        self, executor: ThreadPoolExecutor, thread_name_prefix: Optional[str] = None
+    ) -> None:
+        """
+        Configure a ThreadPoolExecutor instance.
+
+        This method ensures that a ThreadPoolExecutor is correctly set for downloads by adjusting its
+        maximum number of workers if necessary. It also configures the thread name prefix to identify
+        threads created by the executor, which is useful for distinguishing between executors created
+        for parallel product downloads versus those created for other purposes.
+
+        :param executor: A ThreadPoolExecutor instance.
+        :param thread_name_prefix: (optional) A prefix for naming threads created by the executor.
+                                   When provided, threads will be named using this prefix to help
+                                   identify the executor's purpose (e.g., "eodag-download-all").
+        """
+        if (
+            max_workers := getattr(self.config, "max_workers", executor._max_workers)
+        ) < executor._max_workers:
+            executor._max_workers = max_workers
+
+        if thread_name_prefix:
+            executor._thread_name_prefix = "eodag-download-all"