eodag 3.10.0__py3-none-any.whl → 4.0.0a1__py3-none-any.whl

eodag/plugins/download/http.py

@@ -92,7 +92,6 @@ if TYPE_CHECKING:
     from eodag.api.product import Asset, EOProduct  # type: ignore
     from eodag.api.search_result import SearchResult
     from eodag.config import PluginConfig
-    from eodag.types import S3SessionKwargs
     from eodag.types.download_args import DownloadConf
     from eodag.utils import DownloadedCallback, Unpack
 
@@ -119,7 +118,8 @@ class HTTPDownload(Download):
           default: ``1``
         * :attr:`~eodag.config.PluginConfig.flatten_top_dirs` (``bool``): if the directory structure should be
           flattened; default: ``True``
-        * :attr:`~eodag.config.PluginConfig.ignore_assets` (``bool``): ignore assets and download using downloadLink;
+        * :attr:`~eodag.config.PluginConfig.ignore_assets` (``bool``): ignore assets and download using
+          eodag:download_link;
           default: ``False``
         * :attr:`~eodag.config.PluginConfig.timeout` (``int``): time to wait until request timeout in seconds;
           default: ``5``
@@ -128,8 +128,8 @@ class HTTPDownload(Download):
         * :attr:`~eodag.config.PluginConfig.no_auth_download` (``bool``): if the download should be done without
           authentication; default: ``True``
         * :attr:`~eodag.config.PluginConfig.order_enabled` (``bool``): if the product has to be ordered to download it;
-          if this parameter is set to true, a mapping for the orderLink has to be added to the metadata mapping of
-          the search plugin used for the provider; default: ``False``
+          if this parameter is set to true, a mapping for ``eodag:order_link`` has to be added to the metadata mapping
+          of the search plugin used for the provider; default: ``False``
         * :attr:`~eodag.config.PluginConfig.order_method` (``str``): HTTP request method for the order request (``GET``
           or ``POST``); default: ``GET``
         * :attr:`~eodag.config.PluginConfig.order_headers` (``[dict[str, str]]``): headers to be added to the order
@@ -140,9 +140,9 @@ class HTTPDownload(Download):
         * :attr:`~eodag.config.PluginConfig.order_status` (:class:`~eodag.config.PluginConfig.OrderStatus`):
           configuration to handle the order status; contains information which method to use, how the response data is
           interpreted, which status corresponds to success, ordered and error and what should be done on success.
-        * :attr:`~eodag.config.PluginConfig.products` (``dict[str, dict[str, Any]``): product type specific config; the
-          keys are the product types, the values are dictionaries which can contain the key
-          :attr:`~eodag.config.PluginConfig.extract` to overwrite the provider config for a specific product type
+        * :attr:`~eodag.config.PluginConfig.products` (``dict[str, dict[str, Any]``): collection specific config; the
+          keys are the collections, the values are dictionaries which can contain the key
+          :attr:`~eodag.config.PluginConfig.extract` to overwrite the provider config for a specific collection
 
     """
 
@@ -158,12 +158,12 @@ class HTTPDownload(Download):
 
         """Send product order request.
 
-        It will be executed once before the download retry loop, if the product is OFFLINE
-        and has `orderLink` in its properties.
+        It will be executed once before the download retry loop, if the product is orderable
+        and has ``eodag:order_link`` in its properties.
         Product ordering can be configured using the following download plugin parameters:
 
             - :attr:`~eodag.config.PluginConfig.order_enabled`: Wether order is enabled or not (may not use this method
-              if no `orderLink` exists)
+              if no ``eodag:order_link`` exists)
 
             - :attr:`~eodag.config.PluginConfig.order_method`: (optional) HTTP request method, GET (default) or POST
 
@@ -174,14 +174,14 @@ class HTTPDownload(Download):
 
         Product properties used for order:
 
-            - **orderLink**: order request URL
+            - **eodag:order_link**: order request URL
 
         :param product: The EO product to order
         :param auth: (optional) authenticated object
         :param kwargs: download additional kwargs
         :returns: the returned json status response
         """
-        product.properties["storageStatus"] = STAGING_STATUS
+        product.properties["order:status"] = STAGING_STATUS
 
         order_method = getattr(self.config, "order_method", "GET").upper()
         ssl_verify = getattr(self.config, "ssl_verify", True)
@@ -192,7 +192,7 @@ class HTTPDownload(Download):
         order_kwargs: OrderKwargs = {}
         if order_method == "POST":
             # separate url & parameters
-            parts = urlparse(str(product.properties["orderLink"]))
+            parts = urlparse(str(product.properties["eodag:order_link"]))
             query_dict = {}
             # `parts.query` may be a JSON with query strings as one of values. If `parse_qs` is executed as first step,
             # the resulting `query_dict` would be erroneous.
@@ -205,7 +205,7 @@ class HTTPDownload(Download):
             if query_dict:
                 order_kwargs["json"] = query_dict
         else:
-            order_url = product.properties["orderLink"]
+            order_url = product.properties["eodag:order_link"]
             order_kwargs = {}
 
         headers = {**getattr(self.config, "order_headers", {}), **USER_AGENT}
@@ -224,7 +224,7 @@ class HTTPDownload(Download):
                     response.raise_for_status()
                     ordered_message = response.text
                     logger.debug(ordered_message)
-                    product.properties["storageStatus"] = STAGING_STATUS
+                    product.properties["order:status"] = STAGING_STATUS
                 except RequestException as e:
                     self._check_auth_exception(e)
                     msg = f"{product.properties['title']} could not be ordered"
@@ -269,16 +269,16 @@ class HTTPDownload(Download):
         # the job id becomes the product id for EcmwfSearch products
         if "ORDERABLE" in product.properties.get("id", ""):
             product.properties["id"] = product.properties.get(
-                "orderId", product.properties["id"]
+                "eodag:order_id", product.properties["id"]
             )
             product.properties["title"] = (
-                (product.product_type or product.provider).upper()
+                (product.collection or product.provider).upper()
                 + "_"
                 + product.properties["id"]
             )
-        if "downloadLink" in product.properties:
+        if "eodag:download_link" in product.properties:
             product.remote_location = product.location = product.properties[
-                "downloadLink"
+                "eodag:download_link"
             ]
             logger.debug(f"Product location updated to {product.location}")
 
@@ -298,7 +298,7 @@ class HTTPDownload(Download):
 
         Product properties used for order status:
 
-            - **orderStatusLink**: order status request URL
+            - **eodag:status_link**: order status request URL
 
         :param product: The ordered EO product
         :param auth: (optional) authenticated object
@@ -355,14 +355,14 @@ class HTTPDownload(Download):
 
         if status_request_method == "POST":
             # separate url & parameters
-            parts = urlparse(str(product.properties["orderStatusLink"]))
+            parts = urlparse(str(product.properties["eodag:status_link"]))
             status_url = parts._replace(query="").geturl()
             query_dict = parse_qs(parts.query)
             if not query_dict and parts.query:
                 query_dict = geojson.loads(parts.query)
             json_data = query_dict if query_dict else None
         else:
-            status_url = product.properties["orderStatusLink"]
+            status_url = product.properties["eodag:status_link"]
             json_data = None
 
         # check header for success before full status request
@@ -443,8 +443,8 @@ class HTTPDownload(Download):
             )
 
             # display progress percentage
-            if "percent" in status_dict:
-                status_percent = str(status_dict["percent"])
+            if "eodag:order_percent" in status_dict:
+                status_percent = str(status_dict["eodag:order_percent"])
                 if status_percent.isdigit():
                     status_percent += "%"
                 logger.info(
@@ -455,9 +455,11 @@ class HTTPDownload(Download):
                 {k: v for k, v in status_dict.items() if v != NOT_AVAILABLE}
             )
 
-            product.properties["orderStatus"] = status_dict.get("status")
+            product.properties["eodag:order_status"] = status_dict.get(
+                "eodag:order_status"
+            )
 
-            status_message = status_dict.get("message")
+            status_message = status_dict.get("eodag:order_message")
 
             # handle status error
             errors: dict[str, Any] = status_config.get("error", {})
@@ -466,16 +468,18 @@ class HTTPDownload(Download):
                     f"Provider {product.provider} returned: {status_dict.get('error_message', status_message)}"
                 )
 
-        product.properties["storageStatus"] = STAGING_STATUS
+        product.properties["order:status"] = STAGING_STATUS
 
-        success_status: dict[str, Any] = status_config.get("success", {}).get("status")
+        success_status: dict[str, Any] = status_config.get("success", {}).get(
+            "eodag:order_status"
+        )
         # if not success
-        if (success_status and success_status != status_dict.get("status")) or (
-            success_code and success_code != response.status_code
-        ):
+        if (
+            success_status and success_status != status_dict.get("eodag:order_status")
+        ) or (success_code and success_code != response.status_code):
             return None
 
-        product.properties["storageStatus"] = ONLINE_STATUS
+        product.properties["order:status"] = ONLINE_STATUS
 
         if not config_on_success:
             # Nothing left to do
@@ -483,9 +487,13 @@ class HTTPDownload(Download):
 
         # need search on success ?
         if config_on_success.get("need_search"):
-            logger.debug(f"Search for new location: {product.properties['searchLink']}")
+            logger.debug(
+                f"Search for new location: {product.properties['eodag:search_link']}"
+            )
             try:
-                response = _request(product.properties["searchLink"], timeout=timeout)
+                response = _request(
+                    product.properties["eodag:search_link"], timeout=timeout
+                )
             except RequestException as e:
                 logger.warning(
                     "%s order status could not be checked, request returned %s",
@@ -503,11 +511,7 @@ class HTTPDownload(Download):
 
         on_success_mm_querypath = (
             # append product.properties as input for on success response parsing
-            mtd_cfg_as_conversion_and_querypath(
-                dict(
-                    {k: str(v) for k, v in product.properties.items()}, **on_success_mm
-                ),
-            )
+            mtd_cfg_as_conversion_and_querypath(on_success_mm)
             if on_success_mm
             else {}
         )
@@ -530,7 +534,7 @@ class HTTPDownload(Download):
                 if len(results) != 1:
                     raise DownloadError(
                         "Could not get a single result after order success for "
-                        f"{product.properties['searchLink']} request. "
+                        f"{product.properties['eodag:search_link']} request. "
                         f"Please search and download {product} again"
                     )
                 assert isinstance(results, list), "results must be in a list"
@@ -581,9 +585,9 @@ class HTTPDownload(Download):
 
         # update product
         product.properties.update(properties_update)
-        if "downloadLink" in properties_update:
+        if "eodag:download_link" in properties_update:
             product.location = product.remote_location = product.properties[
-                "downloadLink"
+                "eodag:download_link"
             ]
         else:
             self.order_response_process(response, product)
@@ -591,7 +595,7 @@ class HTTPDownload(Download):
     def download(
         self,
         product: EOProduct,
-        auth: Optional[Union[AuthBase, S3SessionKwargs, S3ServiceResource]] = None,
+        auth: Optional[Union[AuthBase, S3ServiceResource]] = None,
         progress_callback: Optional[ProgressCallback] = None,
         wait: float = DEFAULT_DOWNLOAD_WAIT,
         timeout: float = DEFAULT_DOWNLOAD_TIMEOUT,
@@ -677,7 +681,7 @@ class HTTPDownload(Download):
                     raise DownloadError(f"product {product.properties['id']} is empty")
                 else:
                     # make sure storage status is online
-                    product.properties["storageStatus"] = ONLINE_STATUS
+                    product.properties["order:status"] = ONLINE_STATUS
 
                 return path
             else:
@@ -720,14 +724,14 @@ class HTTPDownload(Download):
         stream_size = int(self.stream.headers.get("content-length", 0))
         if (
             stream_size == 0
-            and "storageStatus" in product.properties
-            and product.properties["storageStatus"] != ONLINE_STATUS
+            and "order:status" in product.properties
+            and product.properties["order:status"] != ONLINE_STATUS
         ):
             raise NotAvailableError(
                 "%s(initially %s) ordered, got: %s"
                 % (
                     product.properties["title"],
-                    product.properties["storageStatus"],
+                    product.properties["order:status"],
                     self.stream.reason,
                 )
             )
@@ -755,7 +759,7 @@ class HTTPDownload(Download):
     def _stream_download_dict(
         self,
         product: EOProduct,
-        auth: Optional[Union[AuthBase, S3SessionKwargs, S3ServiceResource]] = None,
+        auth: Optional[Union[AuthBase, S3ServiceResource]] = None,
         byte_range: tuple[Optional[int], Optional[int]] = (None, None),
         compress: Literal["zip", "raw", "auto"] = "auto",
         wait: float = DEFAULT_DOWNLOAD_WAIT,
@@ -882,7 +886,7 @@ class HTTPDownload(Download):
             e.response.text.strip() if e is not None and e.response is not None else ""
         )
         # product not available
-        if product.properties.get("storageStatus", ONLINE_STATUS) != ONLINE_STATUS:
+        if product.properties.get("order:status", ONLINE_STATUS) != ONLINE_STATUS:
             msg = (
                 ordered_message
                 if ordered_message and not response_text
@@ -893,7 +897,7 @@ class HTTPDownload(Download):
                 "%s(initially %s) requested, returned: %s"
                 % (
                     product.properties["title"],
-                    product.properties["storageStatus"],
+                    product.properties["order:status"],
                     msg,
                 )
             )
@@ -915,22 +919,22 @@ class HTTPDownload(Download):
         auth: Optional[AuthBase],
     ) -> None:
         if (
-            "orderLink" in product.properties
-            and product.properties.get("storageStatus") == OFFLINE_STATUS
-            and not product.properties.get("orderStatus")
+            "eodag:order_link" in product.properties
+            and product.properties.get("order:status") == OFFLINE_STATUS
+            and not product.properties.get("eodag:order_status")
         ):
             self._order(product=product, auth=auth)
 
         if (
-            product.properties.get("orderStatusLink")
-            and product.properties.get("storageStatus") != ONLINE_STATUS
+            product.properties.get("eodag:status_link")
+            and product.properties.get("order:status") != ONLINE_STATUS
         ):
             self._order_status(product=product, auth=auth)
 
     def order(
         self,
         product: EOProduct,
-        auth: Optional[Union[AuthBase, S3SessionKwargs]] = None,
+        auth: Optional[Union[AuthBase, S3ServiceResource]] = None,
         wait: float = DEFAULT_DOWNLOAD_WAIT,
         timeout: float = DEFAULT_DOWNLOAD_TIMEOUT,
     ) -> None:
@@ -982,7 +986,7 @@ class HTTPDownload(Download):
         )
 
         req_method = (
-            product.properties.get("downloadMethod", "").lower()
+            product.properties.get("eodag:download_method", "").lower()
             or getattr(self.config, "method", "GET").lower()
         )
         url = product.remote_location
@@ -1041,7 +1045,7 @@ class HTTPDownload(Download):
             ).get(
                 "http_code"
             ):
-                product.properties["storageStatus"] = "ORDERED"
+                product.properties["order:status"] = "ORDERED"
                 self._process_exception(None, product, ordered_message)
             stream_size = self._check_stream_size(product) or None
 
@@ -1096,9 +1100,7 @@ class HTTPDownload(Download):
             assets_common_subdir = os.path.commonpath(asset_rel_paths_list)
 
         # product conf overrides provider conf for "flatten_top_dirs"
-        product_conf = getattr(self.config, "products", {}).get(
-            product.product_type, {}
-        )
+        product_conf = getattr(self.config, "products", {}).get(product.collection, {})
         flatten_top_dirs = product_conf.get(
             "flatten_top_dirs", getattr(self.config, "flatten_top_dirs", True)
         )
@@ -1246,9 +1248,7 @@ class HTTPDownload(Download):
             os.makedirs(fs_dir_path)
 
         # product conf overrides provider conf for "flatten_top_dirs"
-        product_conf = getattr(self.config, "products", {}).get(
-            product.product_type, {}
-        )
+        product_conf = getattr(self.config, "products", {}).get(product.collection, {})
         flatten_top_dirs = product_conf.get(
             "flatten_top_dirs", getattr(self.config, "flatten_top_dirs", True)
         )
@@ -1413,7 +1413,7 @@ class HTTPDownload(Download):
     def download_all(
         self,
         products: SearchResult,
-        auth: Optional[Union[AuthBase, S3SessionKwargs]] = None,
+        auth: Optional[Union[AuthBase, S3ServiceResource]] = None,
         downloaded_callback: Optional[DownloadedCallback] = None,
         progress_callback: Optional[ProgressCallback] = None,
         wait: float = DEFAULT_DOWNLOAD_WAIT,

eodag/plugins/manager.py

@@ -36,7 +36,7 @@ from eodag.plugins.base import EODAGPluginMount
 from eodag.plugins.crunch.base import Crunch
 from eodag.plugins.download.base import Download
 from eodag.plugins.search.base import Search
-from eodag.utils import GENERIC_PRODUCT_TYPE, deepcopy, dict_md5sum
+from eodag.utils import GENERIC_COLLECTION, deepcopy, dict_md5sum
 from eodag.utils.exceptions import (
     AuthenticationError,
     MisconfiguredError,
@@ -50,7 +50,6 @@ if TYPE_CHECKING:
     from eodag.api.product import EOProduct
     from eodag.config import PluginConfig, ProviderConfig
     from eodag.plugins.base import PluginTopic
-    from eodag.types import S3SessionKwargs
 
 
 logger = logging.getLogger("eodag.plugins.manager")
@@ -74,7 +73,7 @@ class PluginManager:
 
     supported_topics = set(PLUGINS_TOPICS_KEYS)
 
-    product_type_to_provider_config_map: dict[str, list[ProviderConfig]]
+    collection_to_provider_config_map: dict[str, list[ProviderConfig]]
 
     skipped_plugins: list[str]
 
@@ -134,12 +133,12 @@ class PluginManager:
         if providers_config is not None:
             self.providers_config = providers_config
 
-        self.build_product_type_to_provider_config_map()
+        self.build_collection_to_provider_config_map()
         self._built_plugins_cache: dict[tuple[str, str, str], Any] = {}
 
-    def build_product_type_to_provider_config_map(self) -> None:
-        """Build mapping conf between product types and providers"""
-        self.product_type_to_provider_config_map = {}
+    def build_collection_to_provider_config_map(self) -> None:
+        """Build mapping conf between collections and providers"""
+        self.collection_to_provider_config_map = {}
         for provider in list(self.providers_config):
             provider_config = self.providers_config[provider]
             if not hasattr(provider_config, "products") or not provider_config.products:
@@ -154,26 +153,26 @@ class PluginManager:
             if getattr(provider_config, "priority", None) is None:
                 self.providers_config[provider].priority = provider_config.priority = 0
 
-            for product_type in provider_config.products:
-                product_type_providers = (
-                    self.product_type_to_provider_config_map.setdefault(  # noqa
-                        product_type, []
+            for collection in provider_config.products:
+                collection_providers = (
+                    self.collection_to_provider_config_map.setdefault(  # noqa
+                        collection, []
                     )
                 )
-                product_type_providers.append(provider_config)
-                product_type_providers.sort(key=attrgetter("priority"), reverse=True)
+                collection_providers.append(provider_config)
+                collection_providers.sort(key=attrgetter("priority"), reverse=True)
 
     def get_search_plugins(
-        self, product_type: Optional[str] = None, provider: Optional[str] = None
+        self, collection: Optional[str] = None, provider: Optional[str] = None
     ) -> Iterator[Union[Search, Api]]:
-        """Build and return all the search plugins supporting the given product type,
+        """Build and return all the search plugins supporting the given collection,
         ordered by highest priority, or the search plugin of the given provider
 
-        :param product_type: (optional) The product type that the constructed plugins
+        :param collection: (optional) The collection that the constructed plugins
                              must support
         :param provider: (optional) The provider or the provider group on which to get
             the search plugins
-        :returns: All the plugins supporting the product type, one by one (a generator
+        :returns: All the plugins supporting the collection, one by one (a generator
                   object)
             or :class:`~eodag.plugins.download.Api`)
         :raises: :class:`~eodag.utils.exceptions.UnsupportedProvider`
@@ -196,13 +195,13 @@ class PluginManager:
             return plugin
 
         configs: Optional[list[ProviderConfig]]
-        if product_type:
-            configs = self.product_type_to_provider_config_map.get(product_type)
+        if collection:
+            configs = self.collection_to_provider_config_map.get(collection)
             if not configs:
                 logger.info(
-                    "UnsupportedProductType: %s, using generic settings", product_type
+                    "UnsupportedCollection: %s, using generic settings", collection
                 )
-                configs = self.product_type_to_provider_config_map[GENERIC_PRODUCT_TYPE]
+                configs = self.collection_to_provider_config_map[GENERIC_COLLECTION]
         else:
             configs = list(self.providers_config.values())
 
@@ -211,9 +210,9 @@ class PluginManager:
                 c for c in configs if provider in [getattr(c, "group", None), c.name]
             ]
 
-        if not configs and product_type:
+        if not configs and collection:
             raise UnsupportedProvider(
-                f"{provider} is not (yet) supported for {product_type}"
+                f"{provider} is not (yet) supported for {collection}"
             )
         if not configs:
             raise UnsupportedProvider(f"{provider} is not (yet) supported")
@@ -265,8 +264,8 @@ class PluginManager:
             matching_url = next(iter(product.assets.values()))["href"]
         elif product is not None:
             matching_url = product.properties.get(
-                "downloadLink"
-            ) or product.properties.get("orderLink")
+                "eodag:download_link"
+            ) or product.properties.get("eodag:order_link")
         else:
             # search auth
             matching_url = getattr(associated_plugin.config, "api_endpoint", None)
@@ -289,7 +288,7 @@ class PluginManager:
         matching_url: Optional[str] = None,
         matching_conf: Optional[PluginConfig] = None,
     ) -> Iterator[Authentication]:
-        """Build and return the authentication plugin for the given product_type and
+        """Build and return the authentication plugin for the given collection and
         provider
 
         :param provider: The provider for which to get the authentication plugin
@@ -362,7 +361,7 @@ class PluginManager:
         provider: str,
         matching_url: Optional[str] = None,
         matching_conf: Optional[PluginConfig] = None,
-    ) -> Optional[Union[AuthBase, S3SessionKwargs, S3ServiceResource]]:
+    ) -> Optional[Union[AuthBase, S3ServiceResource]]:
         """Authenticate and return the authenticated object for the first matching
         authentication plugin
 
@@ -400,7 +399,7 @@ class PluginManager:
 
     def sort_providers(self) -> None:
         """Sort providers taking into account current priority order"""
-        for provider_configs in self.product_type_to_provider_config_map.values():
+        for provider_configs in self.collection_to_provider_config_map.values():
             provider_configs.sort(key=attrgetter("priority"), reverse=True)
 
     def set_priority(self, provider: str, priority: int) -> None:
@@ -414,7 +413,7 @@ class PluginManager:
         for (
             _,
             provider_configs,
-        ) in self.product_type_to_provider_config_map.items():
+        ) in self.collection_to_provider_config_map.items():
             for config in provider_configs:
                 if config.name == provider:
                     config.priority = priority

eodag/plugins/search/__init__.py

@@ -30,17 +30,16 @@ if TYPE_CHECKING:
     from requests.auth import AuthBase
 
     from eodag.plugins.authentication.base import Authentication
-    from eodag.types import S3SessionKwargs
 
 
 @dataclass
 class PreparedSearch:
     """An object collecting needed information for search."""
 
-    product_type: Optional[str] = None
+    collection: Optional[str] = None
     page: Optional[int] = DEFAULT_PAGE
     items_per_page: Optional[int] = DEFAULT_ITEMS_PER_PAGE
-    auth: Optional[Union[AuthBase, S3SessionKwargs, S3ServiceResource]] = None
+    auth: Optional[Union[AuthBase, S3ServiceResource]] = None
     auth_plugin: Optional[Authentication] = None
     count: bool = True
     url: Optional[str] = None
@@ -51,6 +50,6 @@ class PreparedSearch:
     query_params: dict[str, Any] = field(init=False, repr=False)
     query_string: str = field(init=False, repr=False)
     search_urls: list[str] = field(init=False, repr=False)
-    product_type_def_params: dict[str, Any] = field(init=False, repr=False)
+    collection_def_params: dict[str, Any] = field(init=False, repr=False)
     total_items_nb: int = field(init=False, repr=False)
     sort_by_qs: str = field(init=False, repr=False)