eodag 3.0.1__py3-none-any.whl → 3.1.0b1__py3-none-any.whl

eodag/plugins/download/http.py

@@ -27,6 +27,7 @@ from datetime import datetime
 from email.message import Message
 from itertools import chain
 from json import JSONDecodeError
+from pathlib import Path
 from typing import (
     TYPE_CHECKING,
     Any,
@@ -80,8 +81,8 @@ from eodag.utils.exceptions import (
     DownloadError,
     MisconfiguredError,
     NotAvailableError,
-    RequestError,
     TimeOutError,
+    ValidationError,
 )
 
 if TYPE_CHECKING:
@@ -122,8 +123,6 @@ class HTTPDownload(Download):
           default: ``5``
         * :attr:`~eodag.config.PluginConfig.ssl_verify` (``bool``): if the ssl certificates should be verified in
           requests; default: ``True``
-        * :attr:`~eodag.config.PluginConfig.output_extension` (``str``): which extension should be used for the
-          downloaded file
         * :attr:`~eodag.config.PluginConfig.no_auth_download` (``bool``): if the download should be done without
           authentication; default: ``True``
         * :attr:`~eodag.config.PluginConfig.order_enabled` (``bool``): if the product has to be ordered to download it;
@@ -140,16 +139,15 @@ class HTTPDownload(Download):
           configuration to handle the order status; contains information which method to use, how the response data is
           interpreted, which status corresponds to success, ordered and error and what should be done on success.
         * :attr:`~eodag.config.PluginConfig.products` (``Dict[str, Dict[str, Any]``): product type specific config; the
-          keys are the product types, the values are dictionaries which can contain the keys
-          :attr:`~eodag.config.PluginConfig.output_extension` and :attr:`~eodag.config.PluginConfig.extract` to
-          overwrite the provider config for a specific product type
+          keys are the product types, the values are dictionaries which can contain the key
+          :attr:`~eodag.config.PluginConfig.extract` to overwrite the provider config for a specific product type
 
     """
 
     def __init__(self, provider: str, config: PluginConfig) -> None:
         super(HTTPDownload, self).__init__(provider, config)
 
-    def order_download(
+    def _order(
         self,
         product: EOProduct,
         auth: Optional[AuthBase] = None,
@@ -226,9 +224,11 @@ class HTTPDownload(Download):
                     product.properties["storageStatus"] = STAGING_STATUS
                 except RequestException as e:
                     self._check_auth_exception(e)
-                    title = product.properties["title"]
-                    message = f"{title} could not be ordered"
-                    raise RequestError.from_error(e, message) from e
+                    msg = f'{product.properties["title"]} could not be ordered'
+                    if e.response is not None and e.response.status_code == 400:
+                        raise ValidationError.from_error(e, msg) from e
+                    else:
+                        raise DownloadError.from_error(e, msg) from e
 
                 return self.order_response_process(response, product)
         except requests.exceptions.Timeout as exc:
@@ -271,7 +271,7 @@ class HTTPDownload(Download):
 
         return json_response
 
-    def order_download_status(
+    def _order_status(
         self,
         product: EOProduct,
         auth: Optional[AuthBase] = None,
@@ -398,13 +398,11 @@ class HTTPDownload(Download):
                     # success and no need to get status response content
                     skip_parsing_status_response = True
             except RequestException as e:
-                raise DownloadError(
-                    "%s order status could not be checked, request returned %s"
-                    % (
-                        product.properties["title"],
-                        e,
-                    )
-                ) from e
+                msg = f'{product.properties["title"]} order status could not be checked'
+                if e.response is not None and e.response.status_code == 400:
+                    raise ValidationError.from_error(e, msg) from e
+                else:
+                    raise DownloadError.from_error(e, msg) from e
 
         if not skip_parsing_status_response:
             # status request
@@ -566,8 +564,8 @@ class HTTPDownload(Download):
         product: EOProduct,
         auth: Optional[Union[AuthBase, Dict[str, str]]] = None,
         progress_callback: Optional[ProgressCallback] = None,
-        wait: int = DEFAULT_DOWNLOAD_WAIT,
-        timeout: int = DEFAULT_DOWNLOAD_TIMEOUT,
+        wait: float = DEFAULT_DOWNLOAD_WAIT,
+        timeout: float = DEFAULT_DOWNLOAD_TIMEOUT,
         **kwargs: Unpack[DownloadConf],
     ) -> Optional[str]:
         """Download a product using HTTP protocol.
@@ -585,13 +583,6 @@ class HTTPDownload(Download):
             )
             progress_callback = ProgressCallback(disable=True)
 
-        output_extension = getattr(self.config, "products", {}).get(
-            product.product_type, {}
-        ).get("output_extension", None) or getattr(
-            self.config, "output_extension", ".zip"
-        )
-        kwargs["output_extension"] = kwargs.get("output_extension", output_extension)
-
         fs_path, record_filename = self._prepare_download(
             product,
             progress_callback=progress_callback,
@@ -610,7 +601,7 @@ class HTTPDownload(Download):
             try:
                 fs_path = self._download_assets(
                     product,
-                    fs_path.replace(".zip", ""),
+                    fs_path,
                     record_filename,
                     auth,
                     progress_callback,
@@ -627,82 +618,62 @@ class HTTPDownload(Download):
 
         url = product.remote_location
 
-        @self._download_retry(product, wait, timeout)
+        @self._order_download_retry(product, wait, timeout)
         def download_request(
             product: EOProduct,
             auth: AuthBase,
             progress_callback: ProgressCallback,
-            wait: int,
-            timeout: int,
+            wait: float,
+            timeout: float,
             **kwargs: Unpack[DownloadConf],
-        ) -> None:
-            chunks = self._stream_download(product, auth, progress_callback, **kwargs)
+        ) -> os.PathLike:
             is_empty = True
+            chunk_iterator = self._stream_download(
+                product, auth, progress_callback, **kwargs
+            )
+            if fs_path is not None:
+                ext = Path(product.filename).suffix
+                path = Path(fs_path).with_suffix(ext)
 
-            with open(fs_path, "wb") as fhandle:
-                for chunk in chunks:
-                    is_empty = False
-                    fhandle.write(chunk)
+                with open(path, "wb") as fhandle:
+                    for chunk in chunk_iterator:
+                        is_empty = False
+                        progress_callback(len(chunk))
+                        fhandle.write(chunk)
+                self.stream.close()  # Closing response stream
 
-            if is_empty:
-                raise DownloadError(f"product {product.properties['id']} is empty")
+                if is_empty:
+                    raise DownloadError(f"product {product.properties['id']} is empty")
 
-        download_request(product, auth, progress_callback, wait, timeout, **kwargs)
+                return path
+            else:
+                raise DownloadError(
+                    f"download of product {product.properties['id']} failed"
+                )
+
+        path = download_request(
+            product, auth, progress_callback, wait, timeout, **kwargs
+        )
 
         with open(record_filename, "w") as fh:
             fh.write(url)
         logger.debug("Download recorded in %s", record_filename)
 
-        # Check that the downloaded file is really a zip file
-        if not zipfile.is_zipfile(fs_path) and output_extension == ".zip":
-            logger.warning(
-                "Downloaded product is not a Zip File. Please check its file type before using it"
-            )
-            new_fs_path = os.path.join(
-                os.path.dirname(fs_path),
-                sanitize(product.properties["title"]),
-            )
-            if os.path.isfile(fs_path) and not tarfile.is_tarfile(fs_path):
-                if not os.path.isdir(new_fs_path):
-                    os.makedirs(new_fs_path)
-                shutil.move(fs_path, new_fs_path)
-                file_path = os.path.join(new_fs_path, os.path.basename(fs_path))
-                new_file_path = file_path[: file_path.index(".zip")]
-                shutil.move(file_path, new_file_path)
-            # in the case where the outputs extension has not been set
-            # to ".tar" in the product type nor provider configuration
-            elif tarfile.is_tarfile(fs_path):
-                if not new_fs_path.endswith(".tar"):
-                    new_fs_path += ".tar"
-                shutil.move(fs_path, new_fs_path)
-                kwargs["output_extension"] = ".tar"
-                product_path = self._finalize(
-                    new_fs_path,
-                    progress_callback=progress_callback,
-                    **kwargs,
-                )
-                product.location = path_to_uri(product_path)
-                return product_path
-            else:
-                # not a file (dir with zip extension)
-                shutil.move(fs_path, new_fs_path)
-            product.location = path_to_uri(new_fs_path)
-            return new_fs_path
-
-        if os.path.isfile(fs_path) and not (
-            zipfile.is_zipfile(fs_path) or tarfile.is_tarfile(fs_path)
+        if os.path.isfile(path) and not (
+            zipfile.is_zipfile(path) or tarfile.is_tarfile(path)
         ):
             new_fs_path = os.path.join(
-                os.path.dirname(fs_path),
+                os.path.dirname(path),
                 sanitize(product.properties["title"]),
             )
             if not os.path.isdir(new_fs_path):
                 os.makedirs(new_fs_path)
-            shutil.move(fs_path, new_fs_path)
+            shutil.move(path, new_fs_path)
             product.location = path_to_uri(new_fs_path)
             return new_fs_path
+
         product_path = self._finalize(
-            fs_path,
+            str(path),
             progress_callback=progress_callback,
             **kwargs,
         )
@@ -743,15 +714,6 @@ class HTTPDownload(Download):
                     ext = guess_extension(content_type)
                     if ext:
                         filename += ext
-                else:
-                    output_extension: Optional[str] = (
-                        getattr(self.config, "products", {})
-                        .get(product.product_type, {})
-                        .get("output_extension")
-                    )
-                    if output_extension:
-                        filename += output_extension
-
         return filename
 
     def _stream_download_dict(
@@ -759,8 +721,8 @@ class HTTPDownload(Download):
         product: EOProduct,
         auth: Optional[Union[AuthBase, Dict[str, str]]] = None,
         progress_callback: Optional[ProgressCallback] = None,
-        wait: int = DEFAULT_DOWNLOAD_WAIT,
-        timeout: int = DEFAULT_DOWNLOAD_TIMEOUT,
+        wait: float = DEFAULT_DOWNLOAD_WAIT,
+        timeout: float = DEFAULT_DOWNLOAD_TIMEOUT,
         **kwargs: Unpack[DownloadConf],
     ) -> StreamResponse:
         r"""
@@ -834,17 +796,20 @@ class HTTPDownload(Download):
                 else:
                     pass
 
-        chunks = self._stream_download(product, auth, progress_callback, **kwargs)
+        chunk_iterator = self._stream_download(
+            product, auth, progress_callback, **kwargs
+        )
+
         # start reading chunks to set product.headers
         try:
-            first_chunk = next(chunks)
+            first_chunk = next(chunk_iterator)
         except StopIteration:
             # product is empty file
             logger.error("product %s is empty", product.properties["id"])
             raise NotAvailableError(f"product {product.properties['id']} is empty")
 
         return StreamResponse(
-            content=chain(iter([first_chunk]), chunks),
+            content=chain(iter([first_chunk]), chunk_iterator),
             headers=product.headers,
         )
 
@@ -902,6 +867,44 @@ class HTTPDownload(Download):
             else:
                 logger.error("Error while getting resource :\n%s", tb.format_exc())
 
+    def _order_request(
+        self,
+        product: EOProduct,
+        auth: Optional[AuthBase],
+    ) -> None:
+        if (
+            "orderLink" in product.properties
+            and product.properties.get("storageStatus") == OFFLINE_STATUS
+            and not product.properties.get("orderStatus")
+        ):
+            self._order(product=product, auth=auth)
+
+        if (
+            product.properties.get("orderStatusLink", None)
+            and product.properties.get("storageStatus") != ONLINE_STATUS
+        ):
+            self._order_status(product=product, auth=auth)
+
+    def order(
+        self,
+        product: EOProduct,
+        auth: Optional[Union[AuthBase, Dict[str, str]]] = None,
+        wait: float = DEFAULT_DOWNLOAD_WAIT,
+        timeout: float = DEFAULT_DOWNLOAD_TIMEOUT,
+    ) -> None:
+        """
+        Order product and poll to check its status
+
+        :param product: The EO product to download
+        :param auth: (optional) authenticated object
+        :param wait: (optional) Wait time in minutes between two order status check
+        :param timeout: (optional) Maximum time in minutes before stop checking
+                        order status
+        """
+        self._order_download_retry(product, wait, timeout)(self._order_request)(
+            product, auth
+        )
+
     def _stream_download(
         self,
         product: EOProduct,
@@ -910,8 +913,9 @@ class HTTPDownload(Download):
         **kwargs: Unpack[DownloadConf],
     ) -> Iterator[Any]:
         """
-        fetches a zip file containing the assets of a given product as a stream
+        Fetches a zip file containing the assets of a given product as a stream
         and returns a generator yielding the chunks of the file
+
         :param product: product for which the assets should be downloaded
         :param auth: The configuration of a plugin of type Authentication
         :param progress_callback: A method or a callable object
@@ -928,18 +932,8 @@ class HTTPDownload(Download):
         ssl_verify = getattr(self.config, "ssl_verify", True)
 
         ordered_message = ""
-        if (
-            "orderLink" in product.properties
-            and product.properties.get("storageStatus") == OFFLINE_STATUS
-            and not product.properties.get("orderStatus")
-        ):
-            self.order_download(product=product, auth=auth)
-
-        if (
-            product.properties.get("orderStatusLink", None)
-            and product.properties.get("storageStatus") != ONLINE_STATUS
-        ):
-            self.order_download_status(product=product, auth=auth)
+        # retry handled at download level
+        self._order_request(product, auth)
 
         params = kwargs.pop("dl_url_params", None) or getattr(
             self.config, "dl_url_params", {}
@@ -969,7 +963,7 @@ class HTTPDownload(Download):
             auth = None
 
         s = requests.Session()
-        with s.request(
+        self.stream = s.request(
             req_method,
             req_url,
             stream=True,
@@ -979,48 +973,46 @@ class HTTPDownload(Download):
             timeout=DEFAULT_STREAM_REQUESTS_TIMEOUT,
             verify=ssl_verify,
             **req_kwargs,
-        ) as self.stream:
-            try:
-                self.stream.raise_for_status()
-            except requests.exceptions.Timeout as exc:
-                raise TimeOutError(
-                    exc, timeout=DEFAULT_STREAM_REQUESTS_TIMEOUT
-                ) from exc
-            except RequestException as e:
-                self._process_exception(e, product, ordered_message)
-            else:
-                # check if product was ordered
-
-                if getattr(
-                    self.stream, "status_code", None
-                ) is not None and self.stream.status_code == getattr(
-                    self.config, "order_status", {}
-                ).get(
-                    "ordered", {}
-                ).get(
-                    "http_code"
-                ):
-                    product.properties["storageStatus"] = "ORDERED"
-                    self._process_exception(None, product, ordered_message)
-                stream_size = self._check_stream_size(product) or None
-
-                product.headers = self.stream.headers
-                filename = self._check_product_filename(product) or None
-                product.headers[
-                    "content-disposition"
-                ] = f"attachment; filename={filename}"
-                content_type = product.headers.get("Content-Type")
-                guessed_content_type = (
-                    guess_file_type(filename) if filename and not content_type else None
-                )
-                if guessed_content_type is not None:
-                    product.headers["Content-Type"] = guessed_content_type
+        )
+        try:
+            self.stream.raise_for_status()
+        except requests.exceptions.Timeout as exc:
+            raise TimeOutError(exc, timeout=DEFAULT_STREAM_REQUESTS_TIMEOUT) from exc
+        except RequestException as e:
+            self._process_exception(e, product, ordered_message)
+            raise DownloadError(
+                f"download of {product.properties['id']} is empty"
+            ) from e
+        else:
+            # check if product was ordered
+
+            if getattr(
+                self.stream, "status_code", None
+            ) is not None and self.stream.status_code == getattr(
+                self.config, "order_status", {}
+            ).get(
+                "ordered", {}
+            ).get(
+                "http_code"
+            ):
+                product.properties["storageStatus"] = "ORDERED"
+                self._process_exception(None, product, ordered_message)
+            stream_size = self._check_stream_size(product) or None
+
+            product.headers = self.stream.headers
+            filename = self._check_product_filename(product)
+            product.headers["content-disposition"] = f"attachment; filename={filename}"
+            content_type = product.headers.get("Content-Type")
+            guessed_content_type = (
+                guess_file_type(filename) if filename and not content_type else None
+            )
+            if guessed_content_type is not None:
+                product.headers["Content-Type"] = guessed_content_type
 
-                progress_callback.reset(total=stream_size)
-                for chunk in self.stream.iter_content(chunk_size=64 * 1024):
-                    if chunk:
-                        progress_callback(len(chunk))
-                        yield chunk
+            progress_callback.reset(total=stream_size)
+
+            product.filename = filename
+            return self.stream.iter_content(chunk_size=64 * 1024)
 
     def _stream_download_assets(
         self,
@@ -1375,8 +1367,8 @@ class HTTPDownload(Download):
         auth: Optional[Union[AuthBase, Dict[str, str]]] = None,
         downloaded_callback: Optional[DownloadedCallback] = None,
         progress_callback: Optional[ProgressCallback] = None,
-        wait: int = DEFAULT_DOWNLOAD_WAIT,
-        timeout: int = DEFAULT_DOWNLOAD_TIMEOUT,
+        wait: float = DEFAULT_DOWNLOAD_WAIT,
+        timeout: float = DEFAULT_DOWNLOAD_TIMEOUT,
         **kwargs: Unpack[DownloadConf],
     ):
         """

eodag/plugins/download/s3rest.py

@@ -95,8 +95,8 @@ class S3RestDownload(Download):
         product: EOProduct,
         auth: Optional[Union[AuthBase, Dict[str, str]]] = None,
         progress_callback: Optional[ProgressCallback] = None,
-        wait: int = DEFAULT_DOWNLOAD_WAIT,
-        timeout: int = DEFAULT_DOWNLOAD_TIMEOUT,
+        wait: float = DEFAULT_DOWNLOAD_WAIT,
+        timeout: float = DEFAULT_DOWNLOAD_TIMEOUT,
         **kwargs: Unpack[DownloadConf],
     ) -> Optional[str]:
         """Download method for S3 REST API.
@@ -130,9 +130,9 @@ class S3RestDownload(Download):
             and "storageStatus" in product.properties
             and product.properties["storageStatus"] != ONLINE_STATUS
         ):
-            self.http_download_plugin.order_download(product=product, auth=auth)
+            self.http_download_plugin._order(product=product, auth=auth)
 
-        @self._download_retry(product, wait, timeout)
+        @self._order_download_retry(product, wait, timeout)
         def download_request(
             product: EOProduct,
             auth: AuthBase,
@@ -142,9 +142,7 @@ class S3RestDownload(Download):
         ):
             # check order status
             if product.properties.get("orderStatusLink", None):
-                self.http_download_plugin.order_download_status(
-                    product=product, auth=auth
-                )
+                self.http_download_plugin._order_status(product=product, auth=auth)
 
             # get bucket urls
             bucket_name, prefix = get_bucket_name_and_prefix(

eodag/plugins/search/base.py

@@ -31,7 +31,7 @@ from eodag.api.product.metadata_mapping import (
 from eodag.plugins.base import PluginTopic
 from eodag.plugins.search import PreparedSearch
 from eodag.types import model_fields_to_annotated
-from eodag.types.queryables import Queryables
+from eodag.types.queryables import Queryables, QueryablesDict
 from eodag.types.search_args import SortByList
 from eodag.utils import (
     GENERIC_PRODUCT_TYPE,
@@ -325,35 +325,93 @@ class Search(PluginTopic):
                 sort_by_qs += parsed_sort_by_tpl
         return (sort_by_qs, sort_by_qp)
 
+    def _get_product_type_queryables(
+        self, product_type: Optional[str], alias: Optional[str], filters: Dict[str, Any]
+    ) -> Dict[str, Annotated[Any, FieldInfo]]:
+        default_values: Dict[str, Any] = deepcopy(
+            getattr(self.config, "products", {}).get(product_type, {})
+        )
+        default_values.pop("metadata_mapping", None)
+        try:
+            filters["productType"] = product_type
+            return self.discover_queryables(**{**default_values, **filters}) or {}
+        except NotImplementedError:
+            return self.queryables_from_metadata_mapping(product_type, alias)
+
     def list_queryables(
         self,
         filters: Dict[str, Any],
+        available_product_types: List[Any],
+        product_type_configs: Dict[str, Dict[str, Any]],
         product_type: Optional[str] = None,
-    ) -> Dict[str, Annotated[Any, FieldInfo]]:
+        alias: Optional[str] = None,
+    ) -> QueryablesDict:
         """
         Get queryables
 
         :param filters: Additional filters for queryables.
+        :param available_product_types: list of available product types
+        :param product_type_configs: dict containing the product type information for all used product types
         :param product_type: (optional) The product type.
+        :param alias: (optional) alias of the product type
 
         :return: A dictionary containing the queryable properties, associating parameters to their
                 annotated type.
         """
-        default_values: Dict[str, Any] = deepcopy(
-            getattr(self.config, "products", {}).get(product_type, {})
+        additional_info = (
+            "Please select a product type to get the possible values of the parameters!"
+            if not product_type
+            else ""
         )
-        default_values.pop("metadata_mapping", None)
-
-        queryables: Dict[str, Annotated[Any, FieldInfo]] = {}
-        try:
-            queryables = self.discover_queryables(**{**default_values, **filters}) or {}
-        except NotImplementedError:
-            pass
+        if product_type or getattr(self.config, "discover_queryables", {}).get(
+            "fetch_url", ""
+        ):
+            if product_type:
+                self.config.product_type_config = product_type_configs[product_type]
+            queryables = self._get_product_type_queryables(product_type, alias, filters)
+            if getattr(self.config, "discover_queryables", {}).get(
+                "constraints_url", ""
+            ):
+                additional_properties = False
+            else:
+                additional_properties = True
+            return QueryablesDict(
+                additional_properties=additional_properties,
+                additional_information=additional_info,
+                **queryables,
+            )
+        else:
+            all_queryables: Dict[str, Any] = {}
+            for pt in available_product_types:
+                self.config.product_type_config = product_type_configs[pt]
+                pt_queryables = self._get_product_type_queryables(pt, None, filters)
+                # only use key and type because values and defaults will vary between product types
+                pt_queryables_neutral = {
+                    k: Annotated[v.__args__[0], Field(default=None)]
+                    for k, v in pt_queryables.items()
+                }
+                all_queryables.update(pt_queryables_neutral)
+            return QueryablesDict(
+                additional_properties=True,
+                additional_information=additional_info,
+                **all_queryables,
+            )
 
+    def queryables_from_metadata_mapping(
+        self, product_type: Optional[str] = None, alias: Optional[str] = None
+    ) -> Dict[str, Annotated[Any, FieldInfo]]:
+        """
+        Extract queryable parameters from product type metadata mapping.
+        :param product_type: product type id (optional)
+        :param alias: (optional) alias of the product type
+        :returns: dict of annotated queryables
+        """
         metadata_mapping: Dict[str, Any] = deepcopy(
             self.get_metadata_mapping(product_type)
         )
 
+        queryables: Dict[str, Annotated[Any, FieldInfo]] = {}
+
         for param in list(metadata_mapping.keys()):
             if NOT_MAPPED in metadata_mapping[param] or not isinstance(
                 metadata_mapping[param], list
@@ -363,28 +421,18 @@ class Search(PluginTopic):
         eodag_queryables = copy_deepcopy(
             model_fields_to_annotated(Queryables.model_fields)
         )
+        # add default value for product type
+        if alias:
+            eodag_queryables.pop("productType")
+            eodag_queryables["productType"] = Annotated[str, Field(default=alias)]
         for k, v in eodag_queryables.items():
             eodag_queryable_field_info = (
                 get_args(v)[1] if len(get_args(v)) > 1 else None
             )
             if not isinstance(eodag_queryable_field_info, FieldInfo):
                 continue
-            # keep default field info of eodag queryables
-            if k in filters and k in queryables:
-                queryable_field_info = (
-                    get_args(queryables[k])[1]
-                    if len(get_args(queryables[k])) > 1
-                    else None
-                )
-                if not isinstance(queryable_field_info, FieldInfo):
-                    continue
-                queryable_field_info.default = filters[k]
-                continue
-            if k in queryables:
-                continue
             if eodag_queryable_field_info.is_required() or (
                 (eodag_queryable_field_info.alias or k) in metadata_mapping
             ):
                 queryables[k] = v
-
         return queryables