eodag 4.0.0a4__py3-none-any.whl → 4.0.0b1__py3-none-any.whl

eodag/plugins/download/base.py

@@ -29,6 +29,8 @@ from pathlib import Path
 from time import sleep
 from typing import TYPE_CHECKING, Any, Callable, Literal, Optional, TypeVar, Union
 
+from concurrent.futures import ThreadPoolExecutor, as_completed
+
 from eodag.api.product.metadata_mapping import ONLINE_STATUS
 from eodag.plugins.base import PluginTopic
 from eodag.utils import (
@@ -105,6 +107,7 @@ class Download(PluginTopic):
         product: EOProduct,
         auth: Optional[Union[AuthBase, S3ServiceResource]] = None,
         progress_callback: Optional[ProgressCallback] = None,
+        executor: Optional[ThreadPoolExecutor] = None,
         wait: float = DEFAULT_DOWNLOAD_WAIT,
         timeout: float = DEFAULT_DOWNLOAD_TIMEOUT,
         **kwargs: Unpack[DownloadConf],
@@ -115,6 +118,7 @@ class Download(PluginTopic):
         :param product: The EO product to download
         :param auth: (optional) authenticated object
         :param progress_callback: (optional) A progress callback
+        :param executor: (optional) An executor to download assets of ``product`` in parallel if it has any
         :param wait: (optional) If download fails, wait time in minutes between two download tries
         :param timeout: (optional) If download fails, maximum time in minutes before stop retrying
                         to download
@@ -447,6 +451,7 @@ class Download(PluginTopic):
         auth: Optional[Union[AuthBase, S3ServiceResource]] = None,
         downloaded_callback: Optional[DownloadedCallback] = None,
         progress_callback: Optional[ProgressCallback] = None,
+        executor: Optional[ThreadPoolExecutor] = None,
         wait: float = DEFAULT_DOWNLOAD_WAIT,
         timeout: float = DEFAULT_DOWNLOAD_TIMEOUT,
         **kwargs: Unpack[DownloadConf],
@@ -454,7 +459,7 @@ class Download(PluginTopic):
         """
         Base download_all method.
 
-        This specific implementation uses the :meth:`eodag.plugins.download.base.Download.download` method
+        This specific implementation uses the :meth:`~eodag.api.product._product.EOProduct.download` method
         implemented by the plugin to **sequentially** attempt to download products.
 
         :param products: Products to download
@@ -465,6 +470,8 @@ class Download(PluginTopic):
                                     its ``__call__`` method. Will be called each time a product
                                     finishes downloading
         :param progress_callback: (optional) A progress callback
+        :param executor: (optional) An executor to download products in parallel which may
+                         be reused to also download assets of these products in parallel.
         :param wait: (optional) If download fails, wait time in minutes between two download tries
         :param timeout: (optional) If download fails, maximum time in minutes before stop retrying
                         to download
@@ -485,9 +492,15 @@ class Download(PluginTopic):
         stop_time = start_time + timedelta(minutes=timeout)
         nb_products = len(products)
         retry_count = 0
-        # another output for notbooks
+        # another output for notebooks
         nb_info = NotebookWidgets()
 
+        # create an executor if not given
+        executor = ThreadPoolExecutor() if executor is None else executor
+        # set thread name prefix so that the EOProduct download() method can identify
+        # whether the executor was created during parallel product downloads or not
+        self._config_executor(executor, "eodag-download-all")
+
         for product in products:
             product.next_try = start_time
 
@@ -508,53 +521,88 @@ class Download(PluginTopic):
             progress_callback.unit_scale = False
         progress_callback.refresh()
 
+        # anticipate nested tasks to download assets in parallel for at least one product
+        nested_asset_downloads = any(
+            product
+            for product in products
+            if (
+                product.downloader
+                and product.downloader.config.type == "AwsDownload"
+                or len(product.assets) > 0
+                and (
+                    not getattr(self.config, "ignore_assets", False)
+                    or kwargs.get("asset") is not None
+                )
+            )
+        )
+
         with progress_callback as bar:
             while "Loop until all products are download or timeout is reached":
-                # try downloading each product before retry
-                for idx, product in enumerate(products):
+                # try downloading each product in parallel before retry
+
+                # Download products in batches to handle nested tasks to download assets in parallel.
+                # We avoid having less workers in the executor than the number of products to download in parallel
+                # to prevent deadlocks. This could happen by submiting and waiting for a task within a task.
+                # We ensure at least one thread is available for these tasks and at least one product is downloaded
+                # at a time.
+                # If there is only one worker, a specific process at assets download level is used to avoid deadlocks.
+                batch_size = len(products)
+                if nested_asset_downloads and executor._max_workers <= batch_size:
+                    batch_size = max(executor._max_workers - 1, 1)
+
+                products_batch = products[:batch_size]
+                futures = {}
+
+                for idx, product in enumerate(products_batch):
                     if datetime.now() >= product.next_try:
                         products[idx].next_try += timedelta(minutes=wait)
-                        try:
-                            paths.append(
-                                product.download(
-                                    progress_callback=product_progress_callback,
-                                    wait=wait,
-                                    timeout=-1,
-                                    **kwargs,
-                                )
-                            )
+                        future = executor.submit(
+                            product.download,
+                            progress_callback=product_progress_callback,
+                            executor=executor,
+                            wait=wait,
+                            timeout=-1,
+                            **kwargs,  # type: ignore
+                        )
+                        futures[future] = product
 
-                            if downloaded_callback:
-                                downloaded_callback(product)
+                for future in as_completed(futures.keys()):
+                    product = futures[future]
+                    try:
+                        result = future.result()
+                        paths.append(result)
 
-                            # product downloaded, to not retry it
-                            products.remove(product)
-                            bar(1)
+                        if downloaded_callback:
+                            downloaded_callback(product)
 
-                            # reset stop time for next product
-                            stop_time = datetime.now() + timedelta(minutes=timeout)
+                        # product downloaded, to not retry it
+                        products.remove(product)
+                        bar(1)
 
-                        except NotAvailableError as e:
-                            logger.info(e)
-                            continue
+                        # reset stop time for next product
+                        stop_time = datetime.now() + timedelta(minutes=timeout)
 
-                        except (AuthenticationError, MisconfiguredError):
-                            logger.exception(
-                                f"Stopped because of credentials problems with provider {self.provider}"
-                            )
-                            raise
+                    except NotAvailableError as e:
+                        logger.info(e)
+                        continue
 
-                        except (RuntimeError, Exception):
-                            import traceback as tb
+                    except (AuthenticationError, MisconfiguredError):
+                        logger.exception(
+                            f"Stopped because of credentials problems with provider {self.provider}"
+                        )
+                        raise
 
-                            logger.error(
-                                f"A problem occurred during download of product: {product}. "
-                                "Skipping it"
-                            )
-                            logger.debug(f"\n{tb.format_exc()}")
+                    except (RuntimeError, Exception):
+                        import traceback as tb
+
+                        logger.error(
+                            f"A problem occurred during download of product: {product}. "
+                            "Skipping it"
+                        )
+                        logger.debug(f"\n{tb.format_exc()}")
 
-                            # product skipped, to not retry it
-                            products.remove(product)
+                        # product skipped, to not retry it
+                        products.remove(product)
 
                 if (
                     len(products) > 0
@@ -567,6 +615,7 @@ class Download(PluginTopic):
                         f"[Retry #{retry_count}, {nb_products - len(products)}/{nb_products} D/L] "
                         f"Waiting {wait_seconds}s until next download try (retry every {wait}' for {timeout}')"
                     )
+
                     logger.info(info_message)
                     nb_info.display_html(info_message)
                     sleep(wait_seconds + 1)
@@ -579,6 +628,9 @@ class Download(PluginTopic):
                 elif len(products) == 0:
                     break
 
+        # Shutdown executor at the end
+        executor.shutdown(wait=True)
+
         return paths
 
     def _order_download_retry(
@@ -641,8 +693,8 @@ class Download(PluginTopic):
                         )
                         logger.info(not_available_info)
                         # Retry-After info from Response header
-                        if hasattr(self, "stream"):
-                            retry_server_info = self.stream.headers.get(
+                        if hasattr(product, "_stream"):
+                            retry_server_info = product._stream.headers.get(
                                 "Retry-After", ""
                             )
                             if retry_server_info:
@@ -663,8 +715,8 @@ class Download(PluginTopic):
                         )
                         logger.info(not_available_info)
                         # Retry-After info from Response header
-                        if hasattr(self, "stream"):
-                            retry_server_info = self.stream.headers.get(
+                        if hasattr(product, "_stream"):
+                            retry_server_info = product._stream.headers.get(
                                 "Retry-After", ""
                             )
                             if retry_server_info:
@@ -690,3 +742,27 @@ class Download(PluginTopic):
             return download_and_retry
 
         return decorator
+
+    def _config_executor(
+        self, executor: ThreadPoolExecutor, thread_name_prefix: Optional[str] = None
+    ) -> None:
+        """
+        Configure a ThreadPoolExecutor instance.
+
+        This method ensures that a ThreadPoolExecutor is correctly set for downloads by adjusting its
+        maximum number of workers if necessary. It also configures the thread name prefix to identify
+        threads created by the executor, which is useful for distinguishing between executors created
+        for parallel product downloads versus those created for other purposes.
+
+        :param executor: A ThreadPoolExecutor instance.
+        :param thread_name_prefix: (optional) A prefix for naming threads created by the executor.
+                                   When provided, threads will be named using this prefix to help
+                                   identify the executor's purpose (e.g., "eodag-download-all").
+        """
+        if (
+            max_workers := getattr(self.config, "max_workers", executor._max_workers)
+        ) < executor._max_workers:
+            executor._max_workers = max_workers
+
+        if thread_name_prefix:
+            executor._thread_name_prefix = "eodag-download-all"

eodag/plugins/download/http.py

@@ -27,24 +27,17 @@ from email.message import Message
 from itertools import chain
 from json import JSONDecodeError
 from pathlib import Path
-from typing import (
-    TYPE_CHECKING,
-    Any,
-    Iterator,
-    Literal,
-    Optional,
-    TypedDict,
-    Union,
-    cast,
-)
+from typing import TYPE_CHECKING, Any, Iterator, Literal, Optional, Union, cast
 from urllib.parse import parse_qs, urlparse
 
 import geojson
 import requests
+from concurrent.futures import ThreadPoolExecutor, as_completed
 from lxml import etree
 from requests import RequestException
 from requests.auth import AuthBase
 from requests.structures import CaseInsensitiveDict
+from typing_extensions import TypedDict
 from zipstream import ZipStream
 
 from eodag.api.product.metadata_mapping import (
@@ -90,10 +83,9 @@ if TYPE_CHECKING:
     from requests import Response
 
     from eodag.api.product import Asset, EOProduct  # type: ignore
-    from eodag.api.search_result import SearchResult
     from eodag.config import PluginConfig
     from eodag.types.download_args import DownloadConf
-    from eodag.utils import DownloadedCallback, Unpack
+    from eodag.utils import Unpack
 
 logger = logging.getLogger("eodag.download.http")
 
@@ -476,6 +468,8 @@ class HTTPDownload(Download):
         if (
             success_status and success_status != status_dict.get("eodag:order_status")
         ) or (success_code and success_code != response.status_code):
+            # Remove the download link if the order has not been completed or was not successful
+            product.properties.pop("eodag:download_link", None)
             return None
 
         product.properties["order:status"] = ONLINE_STATUS
@@ -596,6 +590,7 @@ class HTTPDownload(Download):
         product: EOProduct,
         auth: Optional[Union[AuthBase, S3ServiceResource]] = None,
         progress_callback: Optional[ProgressCallback] = None,
+        executor: Optional[ThreadPoolExecutor] = None,
         wait: float = DEFAULT_DOWNLOAD_WAIT,
         timeout: float = DEFAULT_DOWNLOAD_TIMEOUT,
         **kwargs: Unpack[DownloadConf],
@@ -637,6 +632,7 @@ class HTTPDownload(Download):
                     record_filename,
                     auth,
                     progress_callback,
+                    executor,
                     **kwargs,
                 )
                 if kwargs.get("asset") is None:
@@ -674,7 +670,7 @@ class HTTPDownload(Download):
                         is_empty = False
                         progress_callback(len(chunk))
                         fhandle.write(chunk)
-                self.stream.close()  # Closing response stream
+                product._stream.close()  # Closing response stream
 
                 if is_empty:
                     raise DownloadError(f"product {product.properties['id']} is empty")
@@ -720,7 +716,7 @@ class HTTPDownload(Download):
         return product_path
 
     def _check_stream_size(self, product: EOProduct) -> int:
-        stream_size = int(self.stream.headers.get("content-length", 0))
+        stream_size = int(product._stream.headers.get("content-length", 0))
         if (
             stream_size == 0
             and "order:status" in product.properties
@@ -731,14 +727,14 @@ class HTTPDownload(Download):
                 % (
                     product.properties["title"],
                     product.properties["order:status"],
-                    self.stream.reason,
+                    product._stream.reason,
                 )
             )
         return stream_size
 
     def _check_product_filename(self, product: EOProduct) -> str:
         filename = None
-        asset_content_disposition = self.stream.headers.get("content-disposition")
+        asset_content_disposition = product._stream.headers.get("content-disposition")
         if asset_content_disposition:
             filename = cast(
                 Optional[str],
@@ -746,7 +742,7 @@ class HTTPDownload(Download):
             )
         if not filename:
             # default filename extracted from path
-            filename = str(os.path.basename(self.stream.url))
+            filename = str(os.path.basename(product._stream.url))
             filename_extension = os.path.splitext(filename)[1]
             if not filename_extension:
                 if content_type := getattr(product, "headers", {}).get("Content-Type"):
@@ -789,15 +785,20 @@ class HTTPDownload(Download):
             not getattr(self.config, "ignore_assets", False)
             or kwargs.get("asset") is not None
         ):
+            executor = ThreadPoolExecutor(
+                max_workers=getattr(self.config, "max_workers", None)
+            )
             try:
                 assets_values = product.assets.get_values(kwargs.get("asset"))
-                assets_stream_list = self._stream_download_assets(
-                    product,
-                    auth,
-                    None,
-                    assets_values=assets_values,
-                    **kwargs,
-                )
+                with executor:
+                    assets_stream_list = self._stream_download_assets(
+                        product,
+                        executor,
+                        auth,
+                        None,
+                        assets_values,
+                        **kwargs,
+                    )
 
                 # single asset
                 if len(assets_stream_list) == 1:
@@ -1009,7 +1010,7 @@ class HTTPDownload(Download):
 
         s = requests.Session()
         try:
-            self.stream = s.request(
+            product._stream = s.request(
                 req_method,
                 req_url,
                 stream=True,
@@ -1024,7 +1025,7 @@ class HTTPDownload(Download):
             # location is not a valid url -> product is not available yet
             raise NotAvailableError("Product is not available yet")
         try:
-            self.stream.raise_for_status()
+            product._stream.raise_for_status()
         except requests.exceptions.Timeout as exc:
             raise TimeOutError(exc, timeout=DEFAULT_STREAM_REQUESTS_TIMEOUT) from exc
         except RequestException as e:
@@ -1036,8 +1037,8 @@ class HTTPDownload(Download):
             # check if product was ordered
 
             if getattr(
-                self.stream, "status_code", None
-            ) is not None and self.stream.status_code == getattr(
+                product._stream, "status_code", None
+            ) is not None and product._stream.status_code == getattr(
                 self.config, "order_status", {}
             ).get(
                 "ordered", {}
@@ -1048,7 +1049,7 @@ class HTTPDownload(Download):
                 self._process_exception(None, product, ordered_message)
             stream_size = self._check_stream_size(product) or None
 
-            product.headers = self.stream.headers
+            product.headers = product._stream.headers
             filename = self._check_product_filename(product)
             content_type = product.headers.get("Content-Type")
             guessed_content_type = (
@@ -1061,11 +1062,12 @@ class HTTPDownload(Download):
             product.size = stream_size
 
             product.filename = filename
-            return self.stream.iter_content(chunk_size=64 * 1024)
+            return product._stream.iter_content(chunk_size=64 * 1024)
 
     def _stream_download_assets(
         self,
         product: EOProduct,
+        executor: ThreadPoolExecutor,
         auth: Optional[AuthBase] = None,
         progress_callback: Optional[ProgressCallback] = None,
         assets_values: list[Asset] = [],
@@ -1082,7 +1084,9 @@ class HTTPDownload(Download):
             self.config, "dl_url_params", {}
         )
 
-        total_size = self._get_asset_sizes(assets_values, auth, params) or None
+        total_size = (
+            self._get_asset_sizes(assets_values, executor, auth, params) or None
+        )
 
         progress_callback.reset(total=total_size)
 
@@ -1188,11 +1192,6 @@ class HTTPDownload(Download):
 
         # Process each asset
         for asset in assets_values:
-            if not asset["href"] or asset["href"].startswith("file:"):
-                logger.info(
-                    f"Local asset detected. Download skipped for {asset['href']}"
-                )
-                continue
             asset_chunks = get_chunks_generator(asset)
             try:
                 # start reading chunks to set assets attributes
@@ -1220,6 +1219,7 @@ class HTTPDownload(Download):
         record_filename: str,
         auth: Optional[AuthBase] = None,
         progress_callback: Optional[ProgressCallback] = None,
+        executor: Optional[ThreadPoolExecutor] = None,
         **kwargs: Unpack[DownloadConf],
     ) -> str:
         """Download product assets if they exist"""
@@ -1227,6 +1227,12 @@ class HTTPDownload(Download):
             logger.info("Progress bar unavailable, please call product.download()")
             progress_callback = ProgressCallback(disable=True)
 
+        # create an executor if not given and anticipate the possible need to shut it down
+        executor, shutdown_executor = (
+            (ThreadPoolExecutor(), True) if executor is None else (executor, False)
+        )
+        self._config_executor(executor)
+
         assets_urls = [
             a["href"] for a in getattr(product, "assets", {}).values() if "href" in a
         ]
@@ -1236,7 +1242,7 @@ class HTTPDownload(Download):
         assets_values = product.assets.get_values(kwargs.get("asset"))
 
         assets_stream_list = self._stream_download_assets(
-            product, auth, progress_callback, assets_values=assets_values, **kwargs
+            product, executor, auth, progress_callback, assets_values, **kwargs
         )
 
         # remove existing incomplete file
@@ -1259,15 +1265,14 @@ class HTTPDownload(Download):
                 local_assets_count += 1
                 continue
 
-        for asset_stream in assets_stream_list:
+        def download_asset(asset_stream: StreamResponse) -> None:
             asset_chunks = asset_stream.content
             asset_path = cast(str, asset_stream.arcname)
             asset_abs_path = os.path.join(fs_dir_path, asset_path)
             asset_abs_path_temp = asset_abs_path + "~"
             # create asset subdir if not exist
             asset_abs_path_dir = os.path.dirname(asset_abs_path)
-            if not os.path.isdir(asset_abs_path_dir):
-                os.makedirs(asset_abs_path_dir)
+            os.makedirs(asset_abs_path_dir, exist_ok=True)
             # remove temporary file
             if os.path.isfile(asset_abs_path_temp):
                 os.remove(asset_abs_path_temp)
@@ -1283,6 +1288,27 @@ class HTTPDownload(Download):
                     os.path.basename(asset_abs_path),
                 )
                 os.rename(asset_abs_path_temp, asset_abs_path)
+            return
+
+        # use parallelization if possible
+        # when products are already downloaded in parallel but the executor has only one worker,
+        # we avoid submitting nested tasks to the executor to prevent deadlocks
+        if (
+            executor._thread_name_prefix == "eodag-download-all"
+            and executor._max_workers == 1
+        ):
+            for asset_stream in assets_stream_list:
+                download_asset(asset_stream)
+        else:
+            futures = (
+                executor.submit(download_asset, asset_stream)
+                for asset_stream in assets_stream_list
+            )
+            [f.result() for f in as_completed(futures)]
+
+        if shutdown_executor:
+            executor.shutdown(wait=True)
+
         # only one local asset
         if local_assets_count == len(assets_urls) and local_assets_count == 1:
             # remove empty {fs_dir_path}
@@ -1336,6 +1362,7 @@ class HTTPDownload(Download):
     def _get_asset_sizes(
         self,
         assets_values: list[Asset],
+        executor: ThreadPoolExecutor,
         auth: Optional[AuthBase],
         params: Optional[dict[str, str]],
         zipped: bool = False,
@@ -1344,8 +1371,11 @@ class HTTPDownload(Download):
 
         timeout = getattr(self.config, "timeout", HTTP_REQ_TIMEOUT)
         ssl_verify = getattr(self.config, "ssl_verify", True)
-        # loop for assets size & filename
-        for asset in assets_values:
+
+        # loop for assets size & filename in parallel
+        def fetch_asset_size(asset: Asset) -> None:
+            nonlocal total_size
+
             if asset["href"] and not asset["href"].startswith("file:"):
                 # HEAD request for size & filename
                 try:
@@ -1407,27 +1437,20 @@ class HTTPDownload(Download):
                             asset.size = int(size_str) if size_str.isdigit() else 0
 
                 total_size += asset.size
-        return total_size
 
-    def download_all(
-        self,
-        products: SearchResult,
-        auth: Optional[Union[AuthBase, S3ServiceResource]] = None,
-        downloaded_callback: Optional[DownloadedCallback] = None,
-        progress_callback: Optional[ProgressCallback] = None,
-        wait: float = DEFAULT_DOWNLOAD_WAIT,
-        timeout: float = DEFAULT_DOWNLOAD_TIMEOUT,
-        **kwargs: Unpack[DownloadConf],
-    ):
-        """
-        Download all using parent (base plugin) method
-        """
-        return super(HTTPDownload, self).download_all(
-            products,
-            auth=auth,
-            downloaded_callback=downloaded_callback,
-            progress_callback=progress_callback,
-            wait=wait,
-            timeout=timeout,
-            **kwargs,
-        )
+        # use parallelization if possible
+        # when products are already downloaded in parallel but the executor has only one worker,
+        # we avoid submitting nested tasks to the executor to prevent deadlocks
+        if (
+            executor._thread_name_prefix == "eodag-download-all"
+            and executor._max_workers == 1
+        ):
+            for asset in assets_values:
+                fetch_asset_size(asset)
+        else:
+            futures = (
+                executor.submit(fetch_asset_size, asset) for asset in assets_values
+            )
+            [f.result() for f in as_completed(futures)]
+
+        return total_size

eodag/plugins/search/base.py

@@ -35,6 +35,7 @@ from eodag.plugins.search import PreparedSearch
 from eodag.types import model_fields_to_annotated
 from eodag.types.queryables import Queryables, QueryablesDict
 from eodag.types.search_args import SortByList
+from eodag.types.stac_metadata import CommonStacMetadata, create_stac_metadata_model
 from eodag.utils import (
     GENERIC_COLLECTION,
     copy_deepcopy,
@@ -358,7 +359,7 @@ class Search(PluginTopic):
             queryables = self.discover_queryables(**{**default_values, **filters}) or {}
         except NotImplementedError as e:
             if str(e):
-                logger.debug(str(e))
+                logger.debug("%s, configured metadata-mapping used", str(e))
             queryables = self.queryables_from_metadata_mapping(collection, alias)
 
         return QueryablesDict(**queryables)
@@ -408,9 +409,10 @@ class Search(PluginTopic):
                 col_queryables = self._get_collection_queryables(col, None, filters)
                 all_queryables.update(col_queryables)
             # reset defaults because they may vary between collections
+            queryables_fields = Queryables.from_stac_models().model_fields
             for k, v in all_queryables.items():
                 v.__metadata__[0].default = getattr(
-                    Queryables.model_fields.get(k, Field(None)), "default", None
+                    queryables_fields.get(k, Field(None)), "default", None
                 )
             return QueryablesDict(
                 additional_properties=auto_discovery,
@@ -468,8 +470,11 @@ class Search(PluginTopic):
             ):
                 del metadata_mapping[param]
 
+        queryables_model = create_stac_metadata_model(
+            base_models=[Queryables, CommonStacMetadata]
+        )
         eodag_queryables = copy_deepcopy(
-            model_fields_to_annotated(Queryables.model_fields)
+            model_fields_to_annotated(queryables_model.model_fields)
         )
         queryables["collection"] = eodag_queryables.pop("collection")
         # add default value for collection