eodag 3.0.1__py3-none-any.whl → 3.1.0__py3-none-any.whl

eodag/plugins/download/s3rest.py

@@ -20,7 +20,7 @@ from __future__ import annotations
 import logging
 import os
 import os.path
-from typing import TYPE_CHECKING, Dict, List, Optional, Union
+from typing import TYPE_CHECKING, Optional, Union
 from xml.dom import minidom
 from xml.parsers.expat import ExpatError
 
@@ -54,6 +54,7 @@ from eodag.utils.exceptions import (
 if TYPE_CHECKING:
     from eodag.api.product import EOProduct
     from eodag.config import PluginConfig
+    from eodag.types import S3SessionKwargs
     from eodag.types.download_args import DownloadConf
     from eodag.utils import Unpack
 
@@ -78,7 +79,7 @@ class S3RestDownload(Download):
         * :attr:`~eodag.config.PluginConfig.order_enabled` (``bool``): whether order is enabled
           or not if product is `OFFLINE`
         * :attr:`~eodag.config.PluginConfig.order_method` (``str``) HTTP request method, ``GET`` (default) or ``POST``
-        * :attr:`~eodag.config.PluginConfig.order_headers` (``[Dict[str, str]]``): order request headers
+        * :attr:`~eodag.config.PluginConfig.order_headers` (``[dict[str, str]]``): order request headers
         * :attr:`~eodag.config.PluginConfig.order_on_response` (:class:`~eodag.config.PluginConfig.OrderOnResponse`):
           a typed dictionary containing the key :attr:`~eodag.config.PluginConfig.OrderOnResponse.metadata_mapping`
           which can be used to add new product properties based on the data in response to the order request
@@ -93,10 +94,10 @@ class S3RestDownload(Download):
     def download(
         self,
         product: EOProduct,
-        auth: Optional[Union[AuthBase, Dict[str, str]]] = None,
+        auth: Optional[Union[AuthBase, S3SessionKwargs]] = None,
         progress_callback: Optional[ProgressCallback] = None,
-        wait: int = DEFAULT_DOWNLOAD_WAIT,
-        timeout: int = DEFAULT_DOWNLOAD_TIMEOUT,
+        wait: float = DEFAULT_DOWNLOAD_WAIT,
+        timeout: float = DEFAULT_DOWNLOAD_TIMEOUT,
         **kwargs: Unpack[DownloadConf],
     ) -> Optional[str]:
         """Download method for S3 REST API.
@@ -130,9 +131,9 @@ class S3RestDownload(Download):
             and "storageStatus" in product.properties
             and product.properties["storageStatus"] != ONLINE_STATUS
         ):
-            self.http_download_plugin.order_download(product=product, auth=auth)
+            self.http_download_plugin._order(product=product, auth=auth)
 
-        @self._download_retry(product, wait, timeout)
+        @self._order_download_retry(product, wait, timeout)
         def download_request(
             product: EOProduct,
             auth: AuthBase,
@@ -142,9 +143,7 @@ class S3RestDownload(Download):
         ):
             # check order status
             if product.properties.get("orderStatusLink", None):
-                self.http_download_plugin.order_download_status(
-                    product=product, auth=auth
-                )
+                self.http_download_plugin._order_status(product=product, auth=auth)
 
             # get bucket urls
             bucket_name, prefix = get_bucket_name_and_prefix(
@@ -272,7 +271,7 @@ class S3RestDownload(Download):
                 os.remove(record_filename)
 
             # total size for progress_callback
-            size_list: List[int] = [
+            size_list: list[int] = [
                 int(node.firstChild.nodeValue)  # type: ignore[attr-defined]
                 for node in xmldoc.getElementsByTagName("Size")
                 if node.firstChild is not None

eodag/plugins/manager.py

@@ -21,18 +21,7 @@ import logging
 import re
 from operator import attrgetter
 from pathlib import Path
-from typing import (
-    TYPE_CHECKING,
-    Any,
-    Dict,
-    Iterator,
-    List,
-    Optional,
-    Tuple,
-    Type,
-    Union,
-    cast,
-)
+from typing import TYPE_CHECKING, Any, Iterator, Optional, Union, cast
 
 import pkg_resources
 
@@ -61,6 +50,7 @@ if TYPE_CHECKING:
     from eodag.api.product import EOProduct
     from eodag.config import PluginConfig, ProviderConfig
     from eodag.plugins.base import PluginTopic
+    from eodag.types import S3SessionKwargs
 
 
 logger = logging.getLogger("eodag.plugins.manager")
@@ -84,11 +74,11 @@ class PluginManager:
 
     supported_topics = set(PLUGINS_TOPICS_KEYS)
 
-    product_type_to_provider_config_map: Dict[str, List[ProviderConfig]]
+    product_type_to_provider_config_map: dict[str, list[ProviderConfig]]
 
-    skipped_plugins: List[str]
+    skipped_plugins: list[str]
 
-    def __init__(self, providers_config: Dict[str, ProviderConfig]) -> None:
+    def __init__(self, providers_config: dict[str, ProviderConfig]) -> None:
         self.skipped_plugins = []
         self.providers_config = providers_config
         # Load all the plugins. This will make all plugin classes of a particular
@@ -144,14 +134,14 @@ class PluginManager:
         self.rebuild()
 
     def rebuild(
-        self, providers_config: Optional[Dict[str, ProviderConfig]] = None
+        self, providers_config: Optional[dict[str, ProviderConfig]] = None
     ) -> None:
         """(Re)Build plugin manager mapping and cache"""
         if providers_config is not None:
             self.providers_config = providers_config
 
         self.build_product_type_to_provider_config_map()
-        self._built_plugins_cache: Dict[Tuple[str, str, str], Any] = {}
+        self._built_plugins_cache: dict[tuple[str, str, str], Any] = {}
 
     def build_product_type_to_provider_config_map(self) -> None:
         """Build mapping conf between product types and providers"""
@@ -211,7 +201,7 @@ class PluginManager:
                 )
             return plugin
 
-        configs: Optional[List[ProviderConfig]]
+        configs: Optional[list[ProviderConfig]]
         if product_type:
             configs = self.product_type_to_provider_config_map.get(product_type)
             if not configs:
@@ -378,7 +368,7 @@ class PluginManager:
         provider: str,
         matching_url: Optional[str] = None,
         matching_conf: Optional[PluginConfig] = None,
-    ) -> Optional[Union[AuthBase, Dict[str, str]]]:
+    ) -> Optional[Union[AuthBase, S3SessionKwargs]]:
         """Authenticate and return the authenticated object for the first matching
         authentication plugin
 
@@ -447,7 +437,7 @@ class PluginManager:
         self,
         provider: str,
         plugin_conf: PluginConfig,
-        topic_class: Type[PluginTopic],
+        topic_class: type[PluginTopic],
     ) -> Union[Api, Search, Download, Authentication, Crunch]:
         """Build the plugin of the given topic with the given plugin configuration and
         registered as the given provider

eodag/plugins/search/__init__.py

@@ -24,11 +24,12 @@ from typing import TYPE_CHECKING
 from eodag.utils import DEFAULT_ITEMS_PER_PAGE, DEFAULT_PAGE
 
 if TYPE_CHECKING:
-    from typing import Any, Dict, List, Optional, Union
+    from typing import Any, Optional, Union
 
     from requests.auth import AuthBase
 
     from eodag.plugins.authentication.base import Authentication
+    from eodag.types import S3SessionKwargs
 
 
 @dataclass
@@ -38,7 +39,7 @@ class PreparedSearch:
     product_type: Optional[str] = None
     page: Optional[int] = DEFAULT_PAGE
     items_per_page: Optional[int] = DEFAULT_ITEMS_PER_PAGE
-    auth: Optional[Union[AuthBase, Dict[str, str]]] = None
+    auth: Optional[Union[AuthBase, S3SessionKwargs]] = None
     auth_plugin: Optional[Authentication] = None
     count: bool = True
     url: Optional[str] = None
@@ -46,9 +47,9 @@ class PreparedSearch:
     exception_message: Optional[str] = None
 
     need_count: bool = field(init=False, repr=False)
-    query_params: Dict[str, Any] = field(init=False, repr=False)
+    query_params: dict[str, Any] = field(init=False, repr=False)
     query_string: str = field(init=False, repr=False)
-    search_urls: List[str] = field(init=False, repr=False)
-    product_type_def_params: Dict[str, Any] = field(init=False, repr=False)
+    search_urls: list[str] = field(init=False, repr=False)
+    product_type_def_params: dict[str, Any] = field(init=False, repr=False)
     total_items_nb: int = field(init=False, repr=False)
     sort_by_qs: str = field(init=False, repr=False)

eodag/plugins/search/base.py

@@ -31,7 +31,7 @@ from eodag.api.product.metadata_mapping import (
 from eodag.plugins.base import PluginTopic
 from eodag.plugins.search import PreparedSearch
 from eodag.types import model_fields_to_annotated
-from eodag.types.queryables import Queryables
+from eodag.types.queryables import Queryables, QueryablesDict
 from eodag.types.search_args import SortByList
 from eodag.utils import (
     GENERIC_PRODUCT_TYPE,
@@ -43,12 +43,13 @@ from eodag.utils import (
 from eodag.utils.exceptions import ValidationError
 
 if TYPE_CHECKING:
-    from typing import Any, Dict, List, Optional, Tuple, Union
+    from typing import Any, Optional, Union
 
     from requests.auth import AuthBase
 
     from eodag.api.product import EOProduct
     from eodag.config import PluginConfig
+    from eodag.types import S3SessionKwargs
 
 logger = logging.getLogger("eodag.search.base")
 
@@ -60,9 +61,9 @@ class Search(PluginTopic):
     :param config: An EODAG plugin configuration
     """
 
-    auth: Union[AuthBase, Dict[str, str]]
+    auth: Union[AuthBase, S3SessionKwargs]
     next_page_url: Optional[str]
-    next_page_query_obj: Optional[Dict[str, Any]]
+    next_page_query_obj: Optional[dict[str, Any]]
     total_items_nb: int
     need_count: bool
     _request: Any  # needed by deprecated load_stac_items
@@ -71,7 +72,7 @@ class Search(PluginTopic):
         super(Search, self).__init__(provider, config)
         # Prepare the metadata mapping
         # Do a shallow copy, the structure is flat enough for this to be sufficient
-        metas: Dict[str, Any] = DEFAULT_METADATA_MAPPING.copy()
+        metas: dict[str, Any] = DEFAULT_METADATA_MAPPING.copy()
         # Update the defaults with the mapping value. This will add any new key
         # added by the provider mapping that is not in the default metadata
         if self.config.metadata_mapping:
@@ -90,7 +91,7 @@ class Search(PluginTopic):
         self,
         prep: PreparedSearch = PreparedSearch(),
         **kwargs: Any,
-    ) -> Tuple[List[EOProduct], Optional[int]]:
+    ) -> tuple[list[EOProduct], Optional[int]]:
         """Implementation of how the products must be searched goes here.
 
         This method must return a tuple with (1) a list of :class:`~eodag.api.product._product.EOProduct` instances
@@ -99,13 +100,13 @@ class Search(PluginTopic):
         """
         raise NotImplementedError("A Search plugin must implement a method named query")
 
-    def discover_product_types(self, **kwargs: Any) -> Optional[Dict[str, Any]]:
+    def discover_product_types(self, **kwargs: Any) -> Optional[dict[str, Any]]:
         """Fetch product types list from provider using `discover_product_types` conf"""
         return None
 
     def discover_queryables(
         self, **kwargs: Any
-    ) -> Optional[Dict[str, Annotated[Any, FieldInfo]]]:
+    ) -> Optional[dict[str, Annotated[Any, FieldInfo]]]:
         """Fetch queryables list from provider using :attr:`~eodag.config.PluginConfig.discover_queryables` conf
 
         :param kwargs: additional filters for queryables (``productType`` and other search
@@ -118,7 +119,7 @@ class Search(PluginTopic):
 
     def _get_defaults_as_queryables(
         self, product_type: str
-    ) -> Dict[str, Annotated[Any, FieldInfo]]:
+    ) -> dict[str, Annotated[Any, FieldInfo]]:
         """
         Return given product type default settings as queryables
 
@@ -128,7 +129,7 @@ class Search(PluginTopic):
         defaults = deepcopy(self.config.products.get(product_type, {}))
         defaults.pop("metadata_mapping", None)
 
-        queryables: Dict[str, Annotated[Any, FieldInfo]] = {}
+        queryables: dict[str, Annotated[Any, FieldInfo]] = {}
         for parameter, value in defaults.items():
             queryables[parameter] = Annotated[type(value), Field(default=value)]
         return queryables
@@ -149,8 +150,8 @@ class Search(PluginTopic):
         )
 
     def get_product_type_def_params(
-        self, product_type: str, **kwargs: Any
-    ) -> Dict[str, Any]:
+        self, product_type: str, format_variables: Optional[dict[str, Any]] = None
+    ) -> dict[str, Any]:
         """Get the provider product type definition parameters and specific settings
 
         :param product_type: the desired product type
@@ -170,7 +171,8 @@ class Search(PluginTopic):
             return {
                 k: v
                 for k, v in format_dict_items(
-                    self.config.products[GENERIC_PRODUCT_TYPE], **kwargs
+                    self.config.products[GENERIC_PRODUCT_TYPE],
+                    **(format_variables or {}),
                 ).items()
                 if v
             }
@@ -200,7 +202,7 @@ class Search(PluginTopic):
 
     def get_metadata_mapping(
         self, product_type: Optional[str] = None
-    ) -> Dict[str, Union[str, List[str]]]:
+    ) -> dict[str, Union[str, list[str]]]:
         """Get the plugin metadata mapping configuration (product type specific if exists)
 
         :param product_type: the desired product type
@@ -212,7 +214,7 @@ class Search(PluginTopic):
             )
         return self.config.metadata_mapping
 
-    def get_sort_by_arg(self, kwargs: Dict[str, Any]) -> Optional[SortByList]:
+    def get_sort_by_arg(self, kwargs: dict[str, Any]) -> Optional[SortByList]:
         """Extract the ``sort_by`` argument from the kwargs or the provider default sort configuration
 
         :param kwargs: Search arguments
@@ -233,7 +235,7 @@ class Search(PluginTopic):
 
     def build_sort_by(
         self, sort_by_arg: SortByList
-    ) -> Tuple[str, Dict[str, List[Dict[str, str]]]]:
+    ) -> tuple[str, dict[str, list[dict[str, str]]]]:
         """Build the sorting part of the query string or body by transforming
         the ``sort_by`` argument into a provider-specific string or dictionary
 
@@ -247,9 +249,9 @@ class Search(PluginTopic):
         sort_by_arg = list(dict.fromkeys(sort_by_arg))
 
         sort_by_qs: str = ""
-        sort_by_qp: Dict[str, Any] = {}
+        sort_by_qp: dict[str, Any] = {}
 
-        provider_sort_by_tuples_used: List[Tuple[str, str]] = []
+        provider_sort_by_tuples_used: list[tuple[str, str]] = []
         for eodag_sort_by_tuple in sort_by_arg:
             eodag_sort_param = eodag_sort_by_tuple[0]
             provider_sort_param = self.config.sort["sort_param_mapping"].get(
@@ -282,7 +284,7 @@ class Search(PluginTopic):
                 if eodag_sort_order == "ASC"
                 else self.config.sort["sort_order_mapping"]["descending"]
             )
-            provider_sort_by_tuple: Tuple[str, str] = (
+            provider_sort_by_tuple: tuple[str, str] = (
                 provider_sort_param,
                 provider_sort_order,
             )
@@ -315,7 +317,7 @@ class Search(PluginTopic):
                 sort_order=provider_sort_by_tuple[1],
             )
             try:
-                parsed_sort_by_tpl_dict: Dict[str, Any] = orjson.loads(
+                parsed_sort_by_tpl_dict: dict[str, Any] = orjson.loads(
                     parsed_sort_by_tpl
                 )
                 sort_by_qp = update_nested_dict(
@@ -325,35 +327,87 @@ class Search(PluginTopic):
                 sort_by_qs += parsed_sort_by_tpl
         return (sort_by_qs, sort_by_qp)
 
+    def _get_product_type_queryables(
+        self, product_type: Optional[str], alias: Optional[str], filters: dict[str, Any]
+    ) -> QueryablesDict:
+        default_values: dict[str, Any] = deepcopy(
+            getattr(self.config, "products", {}).get(product_type, {})
+        )
+        default_values.pop("metadata_mapping", None)
+        try:
+            filters["productType"] = product_type
+            queryables = self.discover_queryables(**{**default_values, **filters}) or {}
+        except NotImplementedError:
+            queryables = self.queryables_from_metadata_mapping(product_type, alias)
+
+        return QueryablesDict(**queryables)
+
     def list_queryables(
         self,
-        filters: Dict[str, Any],
+        filters: dict[str, Any],
+        available_product_types: list[Any],
+        product_type_configs: dict[str, dict[str, Any]],
         product_type: Optional[str] = None,
-    ) -> Dict[str, Annotated[Any, FieldInfo]]:
+        alias: Optional[str] = None,
+    ) -> QueryablesDict:
         """
         Get queryables
 
         :param filters: Additional filters for queryables.
+        :param available_product_types: list of available product types
+        :param product_type_configs: dict containing the product type information for all used product types
         :param product_type: (optional) The product type.
+        :param alias: (optional) alias of the product type
 
         :return: A dictionary containing the queryable properties, associating parameters to their
                 annotated type.
         """
-        default_values: Dict[str, Any] = deepcopy(
-            getattr(self.config, "products", {}).get(product_type, {})
+        additional_info = (
+            "Please select a product type to get the possible values of the parameters!"
+            if not product_type
+            else ""
         )
-        default_values.pop("metadata_mapping", None)
-
-        queryables: Dict[str, Annotated[Any, FieldInfo]] = {}
-        try:
-            queryables = self.discover_queryables(**{**default_values, **filters}) or {}
-        except NotImplementedError:
-            pass
+        if product_type or getattr(self.config, "discover_queryables", {}).get(
+            "fetch_url", ""
+        ):
+            if product_type:
+                self.config.product_type_config = product_type_configs[product_type]
+            queryables = self._get_product_type_queryables(product_type, alias, filters)
+            queryables.additional_information = additional_info
+
+            return queryables
+        else:
+            all_queryables: dict[str, Any] = {}
+            for pt in available_product_types:
+                self.config.product_type_config = product_type_configs[pt]
+                pt_queryables = self._get_product_type_queryables(pt, None, filters)
+                # only use key and type because values and defaults will vary between product types
+                pt_queryables_neutral = {
+                    k: Annotated[v.__args__[0], Field(default=None)]
+                    for k, v in pt_queryables.items()
+                }
+                all_queryables.update(pt_queryables_neutral)
+            return QueryablesDict(
+                additional_properties=True,
+                additional_information=additional_info,
+                **all_queryables,
+            )
 
-        metadata_mapping: Dict[str, Any] = deepcopy(
+    def queryables_from_metadata_mapping(
+        self, product_type: Optional[str] = None, alias: Optional[str] = None
+    ) -> dict[str, Annotated[Any, FieldInfo]]:
+        """
+        Extract queryable parameters from product type metadata mapping.
+        :param product_type: product type id (optional)
+        :param alias: (optional) alias of the product type
+        :returns: dict of annotated queryables
+        """
+        metadata_mapping: dict[str, Any] = deepcopy(
             self.get_metadata_mapping(product_type)
         )
 
+        queryables: dict[str, Annotated[Any, FieldInfo]] = {}
+
         for param in list(metadata_mapping.keys()):
             if NOT_MAPPED in metadata_mapping[param] or not isinstance(
                 metadata_mapping[param], list
@@ -363,28 +417,18 @@ class Search(PluginTopic):
         eodag_queryables = copy_deepcopy(
             model_fields_to_annotated(Queryables.model_fields)
         )
+        # add default value for product type
+        if alias:
+            eodag_queryables.pop("productType")
+            eodag_queryables["productType"] = Annotated[str, Field(default=alias)]
         for k, v in eodag_queryables.items():
             eodag_queryable_field_info = (
                 get_args(v)[1] if len(get_args(v)) > 1 else None
             )
             if not isinstance(eodag_queryable_field_info, FieldInfo):
                 continue
-            # keep default field info of eodag queryables
-            if k in filters and k in queryables:
-                queryable_field_info = (
-                    get_args(queryables[k])[1]
-                    if len(get_args(queryables[k])) > 1
-                    else None
-                )
-                if not isinstance(queryable_field_info, FieldInfo):
-                    continue
-                queryable_field_info.default = filters[k]
-                continue
-            if k in queryables:
-                continue
             if eodag_queryable_field_info.is_required() or (
                 (eodag_queryable_field_info.alias or k) in metadata_mapping
             ):
                 queryables[k] = v
-
         return queryables