eodag 3.0.0b3__py3-none-any.whl → 3.1.0__py3-none-any.whl

eodag/api/core.py

@@ -17,29 +17,34 @@
 # limitations under the License.
 from __future__ import annotations
 
+import datetime
 import logging
 import os
 import re
 import shutil
 import tempfile
+from importlib.metadata import version
+from importlib.resources import files as res_files
 from operator import itemgetter
-from typing import TYPE_CHECKING, Any, Dict, Iterator, List, Optional, Set, Tuple, Union
+from typing import TYPE_CHECKING, Any, Iterator, Optional, Union
 
 import geojson
-import pkg_resources
 import yaml.parser
-from pkg_resources import resource_filename
-from pydantic.fields import FieldInfo
 from whoosh import analysis, fields
 from whoosh.fields import Schema
-from whoosh.index import create_in, exists_in, open_dir
+from whoosh.index import exists_in, open_dir
 from whoosh.qparser import QueryParser
 
-from eodag.api.product.metadata_mapping import mtd_cfg_as_conversion_and_querypath
+from eodag.api.product.metadata_mapping import (
+    ONLINE_STATUS,
+    mtd_cfg_as_conversion_and_querypath,
+)
 from eodag.api.search_result import SearchResult
 from eodag.config import (
+    PLUGINS_TOPICS_KEYS,
     PluginConfig,
     SimpleYamlProxyConfig,
+    credentials_in_auth,
     get_ext_product_types_conf,
     load_default_config,
     load_stac_provider_config,
@@ -48,13 +53,15 @@ from eodag.config import (
     override_config_from_file,
     override_config_from_mapping,
     provider_config_init,
+    share_credentials,
 )
 from eodag.plugins.manager import PluginManager
 from eodag.plugins.search import PreparedSearch
-from eodag.plugins.search.build_search_result import BuildPostSearchResult
+from eodag.plugins.search.build_search_result import MeteoblueSearch
+from eodag.plugins.search.qssearch import PostJsonSearch
 from eodag.types import model_fields_to_annotated
-from eodag.types.queryables import CommonQueryables
-from eodag.types.whoosh import EODAGQueryParser
+from eodag.types.queryables import CommonQueryables, QueryablesDict
+from eodag.types.whoosh import EODAGQueryParser, create_in
 from eodag.utils import (
     DEFAULT_DOWNLOAD_TIMEOUT,
     DEFAULT_DOWNLOAD_WAIT,
@@ -65,7 +72,6 @@ from eodag.utils import (
     HTTP_REQ_TIMEOUT,
     MockResponse,
     _deprecated,
-    copy_deepcopy,
     get_geometry_from_various,
     makedirs,
     obj_md5sum,
@@ -76,7 +82,6 @@ from eodag.utils import (
 from eodag.utils.exceptions import (
     AuthenticationError,
     EodagError,
-    MisconfiguredError,
     NoMatchingProductType,
     PluginImplementationError,
     RequestError,
@@ -96,7 +101,7 @@ if TYPE_CHECKING:
     from eodag.plugins.search.base import Search
     from eodag.types import ProviderSortables
     from eodag.types.download_args import DownloadConf
-    from eodag.utils import Annotated, DownloadedCallback, ProgressCallback, Unpack
+    from eodag.utils import DownloadedCallback, ProgressCallback, Unpack
 
 logger = logging.getLogger("eodag.core")
 
@@ -114,8 +119,8 @@ class EODataAccessGateway:
         user_conf_file_path: Optional[str] = None,
         locations_conf_path: Optional[str] = None,
     ) -> None:
-        product_types_config_path = resource_filename(
-            "eodag", os.path.join("resources/", "product_types.yml")
+        product_types_config_path = os.getenv("EODAG_PRODUCT_TYPES_CFG_FILE") or str(
+            res_files("eodag") / "resources" / "product_types.yml"
         )
         self.product_types_config = SimpleYamlProxyConfig(product_types_config_path)
         self.product_types_config_md5 = obj_md5sum(self.product_types_config.source)
@@ -156,8 +161,8 @@ class EODataAccessGateway:
                 user_conf_file_path = standard_configuration_path
                 if not os.path.isfile(standard_configuration_path):
                     shutil.copy(
-                        resource_filename(
-                            "eodag", os.path.join("resources", "user_conf_template.yml")
+                        str(
+                            res_files("eodag") / "resources" / "user_conf_template.yml"
                         ),
                         standard_configuration_path,
                     )
@@ -166,16 +171,21 @@ class EODataAccessGateway:
         # Second level override: From environment variables
         override_config_from_env(self.providers_config)
 
+        # share credentials between updated plugins confs
+        share_credentials(self.providers_config)
+
         # init updated providers conf
-        stac_provider_config = load_stac_provider_config()
         for provider in self.providers_config.keys():
-            provider_config_init(self.providers_config[provider], stac_provider_config)
+            provider_config_init(
+                self.providers_config[provider],
+                load_stac_provider_config(),
+            )
 
         # re-build _plugins_manager using up-to-date providers_config
         self._plugins_manager.rebuild(self.providers_config)
 
         # store pruned providers configs
-        self._pruned_providers_config: Dict[str, Any] = {}
+        self._pruned_providers_config: dict[str, Any] = {}
         # filter out providers needing auth that have no credentials set
         self._prune_providers_list()
 
@@ -193,13 +203,13 @@ class EODataAccessGateway:
                 locations_conf_path = os.path.join(self.conf_dir, "locations.yml")
                 if not os.path.isfile(locations_conf_path):
                     # copy locations conf file and replace path example
-                    locations_conf_template = resource_filename(
-                        "eodag",
-                        os.path.join("resources", "locations_conf_template.yml"),
+                    locations_conf_template = str(
+                        res_files("eodag") / "resources" / "locations_conf_template.yml"
                     )
-                    with open(locations_conf_template) as infile, open(
-                        locations_conf_path, "w"
-                    ) as outfile:
+                    with (
+                        open(locations_conf_template) as infile,
+                        open(locations_conf_path, "w") as outfile,
+                    ):
                         # The template contains paths in the form of:
                         # /path/to/locations/file.shp
                         path_template = "/path/to/locations/"
@@ -211,15 +221,14 @@ class EODataAccessGateway:
                             outfile.write(line)
                     # copy sample shapefile dir
                     shutil.copytree(
-                        resource_filename("eodag", os.path.join("resources", "shp")),
+                        str(res_files("eodag") / "resources" / "shp"),
                         os.path.join(self.conf_dir, "shp"),
                     )
         self.set_locations_conf(locations_conf_path)
-        self.search_errors: Set = set()
 
     def get_version(self) -> str:
         """Get eodag package version"""
-        return pkg_resources.get_distribution("eodag").version
+        return version("eodag")
 
     def build_index(self) -> None:
         """Build a `Whoosh <https://whoosh.readthedocs.io/en/latest/index.html>`_
@@ -296,13 +305,18 @@ class EODataAccessGateway:
                     product_type, **{"md5": self.product_types_config_md5}
                 )
                 # add to index
-                ix_writer.add_document(
-                    **{
-                        k: v
-                        for k, v in versioned_product_type.items()
-                        if k in product_types_schema.names()
-                    }
-                )
+                try:
+                    ix_writer.add_document(
+                        **{
+                            k: v
+                            for k, v in versioned_product_type.items()
+                            if k in product_types_schema.names()
+                        }
+                    )
+                except TypeError as e:
+                    logger.error(
+                        f"Cannot write product type {product_type['ID']} into index. e={e} product_type={product_type}"
+                    )
             ix_writer.commit()
 
     def set_preferred_provider(self, provider: str) -> None:
@@ -320,7 +334,7 @@ class EODataAccessGateway:
             new_priority = max_priority + 1
             self._plugins_manager.set_priority(provider, new_priority)
 
-    def get_preferred_provider(self) -> Tuple[str, int]:
+    def get_preferred_provider(self) -> tuple[str, int]:
         """Get the provider currently set as the preferred one for searching
         products, along with its priority.
 
@@ -333,14 +347,24 @@ class EODataAccessGateway:
         preferred, priority = max(providers_with_priority, key=itemgetter(1))
         return preferred, priority
 
-    def update_providers_config(self, yaml_conf: str) -> None:
+    def update_providers_config(
+        self,
+        yaml_conf: Optional[str] = None,
+        dict_conf: Optional[dict[str, Any]] = None,
+    ) -> None:
         """Update providers configuration with given input.
         Can be used to add a provider to existing configuration or update
         an existing one.
 
         :param yaml_conf: YAML formated provider configuration
+        :param dict_conf: provider configuration as dictionary in place of ``yaml_conf``
         """
-        conf_update = yaml.safe_load(yaml_conf)
+        if dict_conf is not None:
+            conf_update = dict_conf
+        elif yaml_conf is not None:
+            conf_update = yaml.safe_load(yaml_conf)
+        else:
+            return None
 
         # restore the pruned configuration
         for provider in list(self._pruned_providers_config.keys()):
@@ -355,9 +379,14 @@ class EODataAccessGateway:
 
         override_config_from_mapping(self.providers_config, conf_update)
 
-        stac_provider_config = load_stac_provider_config()
+        # share credentials between updated plugins confs
+        share_credentials(self.providers_config)
+
         for provider in conf_update.keys():
-            provider_config_init(self.providers_config[provider], stac_provider_config)
+            provider_config_init(
+                self.providers_config[provider],
+                load_stac_provider_config(),
+            )
             setattr(self.providers_config[provider], "product_types_fetched", False)
         # re-create _plugins_manager using up-to-date providers_config
         self._plugins_manager.build_product_type_to_provider_config_map()
@@ -367,12 +396,12 @@ class EODataAccessGateway:
         name: str,
         url: Optional[str] = None,
         priority: Optional[int] = None,
-        search: Dict[str, Any] = {"type": "StacSearch"},
-        products: Dict[str, Any] = {
+        search: dict[str, Any] = {"type": "StacSearch"},
+        products: dict[str, Any] = {
             GENERIC_PRODUCT_TYPE: {"productType": "{productType}"}
         },
-        download: Dict[str, Any] = {"type": "HTTPDownload", "auth_error_code": 401},
-        **kwargs: Dict[str, Any],
+        download: dict[str, Any] = {"type": "HTTPDownload", "auth_error_code": 401},
+        **kwargs: dict[str, Any],
     ):
         """Adds a new provider.
 
@@ -391,7 +420,7 @@ class EODataAccessGateway:
         :param download: Download :class:`~eodag.config.PluginConfig` mapping
         :param kwargs: Additional :class:`~eodag.config.ProviderConfig` mapping
         """
-        conf_dict: Dict[str, Any] = {
+        conf_dict: dict[str, Any] = {
             name: {
                 "url": url,
                 "search": {"type": "StacSearch", **search},
@@ -419,14 +448,11 @@ class EODataAccessGateway:
 
         # api plugin usage: remove unneeded search/download/auth plugin conf
         if conf_dict[name].get("api"):
-            conf_dict[name].pop("search", None)
-            conf_dict[name].pop("download", None)
-            conf_dict[name].pop("auth", None)
+            for k in PLUGINS_TOPICS_KEYS:
+                if k != "api":
+                    conf_dict[name].pop(k, None)
 
-        override_config_from_mapping(self.providers_config, conf_dict)
-        provider_config_init(self.providers_config[name], load_stac_provider_config())
-        setattr(self.providers_config[name], "product_types_fetched", False)
-        self._plugins_manager.build_product_type_to_provider_config_map()
+        self.update_providers_config(dict_conf=conf_dict)
 
         if priority is None:
             self.set_preferred_provider(name)
@@ -452,12 +478,7 @@ class EODataAccessGateway:
 
             # check authentication
             if hasattr(conf, "api") and getattr(conf.api, "need_auth", False):
-                credentials_exist = any(
-                    [
-                        cred is not None
-                        for cred in getattr(conf.api, "credentials", {}).values()
-                    ]
-                )
+                credentials_exist = credentials_in_auth(conf.api)
                 if not credentials_exist:
                     # credentials needed but not found
                     self._pruned_providers_config[provider] = self.providers_config.pop(
@@ -469,7 +490,7 @@ class EODataAccessGateway:
                         provider,
                     )
             elif hasattr(conf, "search") and getattr(conf.search, "need_auth", False):
-                if not hasattr(conf, "auth"):
+                if not hasattr(conf, "auth") and not hasattr(conf, "search_auth"):
                     # credentials needed but no auth plugin was found
                     self._pruned_providers_config[provider] = self.providers_config.pop(
                         provider
@@ -480,11 +501,13 @@ class EODataAccessGateway:
                         provider,
                     )
                     continue
-                credentials_exist = any(
-                    [
-                        cred is not None
-                        for cred in getattr(conf.auth, "credentials", {}).values()
-                    ]
+                credentials_exist = (
+                    hasattr(conf, "search_auth")
+                    and credentials_in_auth(conf.search_auth)
+                ) or (
+                    not hasattr(conf, "search_auth")
+                    and hasattr(conf, "auth")
+                    and credentials_in_auth(conf.auth)
                 )
                 if not credentials_exist:
                     # credentials needed but not found
@@ -541,7 +564,7 @@ class EODataAccessGateway:
             main_locations_config = locations_config[main_key]
 
             logger.info("Locations configuration loaded from %s" % locations_conf_path)
-            self.locations_config: List[Dict[str, Any]] = main_locations_config
+            self.locations_config: list[dict[str, Any]] = main_locations_config
         else:
             logger.info(
                 "Could not load locations configuration from %s" % locations_conf_path
@@ -550,7 +573,7 @@ class EODataAccessGateway:
 
     def list_product_types(
         self, provider: Optional[str] = None, fetch_providers: bool = True
-    ) -> List[Dict[str, Any]]:
+    ) -> list[dict[str, Any]]:
         """Lists supported product types.
 
         :param provider: (optional) The name of a provider that must support the product
@@ -564,7 +587,7 @@ class EODataAccessGateway:
             # First, update product types list if possible
             self.fetch_product_types_list(provider=provider)
 
-        product_types: List[Dict[str, Any]] = []
+        product_types: list[dict[str, Any]] = []
 
         providers_configs = (
             list(self.providers_config.values())
@@ -599,21 +622,32 @@ class EODataAccessGateway:
     def fetch_product_types_list(self, provider: Optional[str] = None) -> None:
         """Fetch product types list and update if needed
 
-        :param provider: (optional) The name of a provider for which product types list
-                         should be updated. Defaults to all providers (None value).
+        :param provider: The name of a provider or provider-group for which product types
+                         list should be updated. Defaults to all providers (None value).
         """
+        providers_to_fetch = list(self.providers_config.keys())
+        # check if some providers are grouped under a group name which is not a provider name
         if provider is not None and provider not in self.providers_config:
-            return
+            providers_to_fetch = [
+                p
+                for p, pconf in self.providers_config.items()
+                if provider == getattr(pconf, "group", None)
+            ]
+            if providers_to_fetch:
+                logger.info(
+                    f"Fetch product types for {provider} group: {', '.join(providers_to_fetch)}"
+                )
+            else:
+                return None
+        elif provider is not None:
+            providers_to_fetch = [provider]
 
         # providers discovery confs that are fetchable
-        providers_discovery_configs_fetchable: Dict[str, Any] = {}
+        providers_discovery_configs_fetchable: dict[str, Any] = {}
         # check if any provider has not already been fetched for product types
         already_fetched = True
-        for provider_to_fetch, provider_config in (
-            {provider: self.providers_config[provider]}.items()
-            if provider
-            else self.providers_config.items()
-        ):
+        for provider_to_fetch in providers_to_fetch:
+            provider_config = self.providers_config[provider_to_fetch]
             # get discovery conf
             if hasattr(provider_config, "search"):
                 provider_search_config = provider_config.search
@@ -732,29 +766,40 @@ class EODataAccessGateway:
 
     def discover_product_types(
         self, provider: Optional[str] = None
-    ) -> Optional[Dict[str, Any]]:
+    ) -> Optional[dict[str, Any]]:
         """Fetch providers for product types
 
-        :param provider: (optional) The name of a provider to fetch. Defaults to all
-                         providers (None value).
+        :param provider: The name of a provider or provider-group to fetch. Defaults to
+                         all providers (None value).
         :returns: external product types configuration
         """
-        if provider and provider not in self.providers_config:
+        grouped_providers = [
+            p
+            for p, provider_config in self.providers_config.items()
+            if provider == getattr(provider_config, "group", None)
+        ]
+        if provider and provider not in self.providers_config and grouped_providers:
+            logger.info(
+                f"Discover product types for {provider} group: {', '.join(grouped_providers)}"
+            )
+        elif provider and provider not in self.providers_config:
             raise UnsupportedProvider(
                 f"The requested provider is not (yet) supported: {provider}"
             )
-        ext_product_types_conf: Dict[str, Any] = {}
+        ext_product_types_conf: dict[str, Any] = {}
         providers_to_fetch = [
             p
             for p in (
                 [
-                    provider,
+                    p
+                    for p in self.providers_config
+                    if p in grouped_providers + [provider]
                 ]
                 if provider
                 else self.available_providers()
             )
         ]
-        kwargs: Dict[str, Any] = {}
+        kwargs: dict[str, Any] = {}
         for provider in providers_to_fetch:
             if hasattr(self.providers_config[provider], "search"):
                 search_plugin_config = self.providers_config[provider].search
@@ -762,7 +807,9 @@ class EODataAccessGateway:
                 search_plugin_config = self.providers_config[provider].api
             else:
                 return None
-            if getattr(search_plugin_config, "discover_product_types", None):
+            if getattr(search_plugin_config, "discover_product_types", {}).get(
+                "fetch_url", None
+            ):
                 search_plugin: Union[Search, Api] = next(
                     self._plugins_manager.get_search_plugins(provider=provider)
                 )
@@ -773,23 +820,15 @@ class EODataAccessGateway:
                     continue
                 # append auth to search plugin if needed
                 if getattr(search_plugin.config, "need_auth", False):
-                    auth_plugin = self._plugins_manager.get_auth_plugin(
-                        search_plugin.provider
-                    )
-                    if auth_plugin and callable(
-                        getattr(auth_plugin, "authenticate", None)
+                    if auth := self._plugins_manager.get_auth(
+                        search_plugin.provider,
+                        getattr(search_plugin.config, "api_endpoint", None),
+                        search_plugin.config,
                     ):
-                        try:
-                            kwargs["auth"] = auth_plugin.authenticate()
-                        except (AuthenticationError, MisconfiguredError) as e:
-                            logger.warning(
-                                f"Could not authenticate on {provider}: {str(e)}"
-                            )
-                            ext_product_types_conf[provider] = None
-                            continue
+                        kwargs["auth"] = auth
                     else:
-                        logger.warning(
-                            f"Could not authenticate on {provider} using {auth_plugin} plugin"
+                        logger.debug(
+                            f"Could not authenticate on {provider} for product types discovery"
                         )
                         ext_product_types_conf[provider] = None
                         continue
@@ -801,7 +840,7 @@ class EODataAccessGateway:
         return sort_dict(ext_product_types_conf)
 
     def update_product_types_list(
-        self, ext_product_types_conf: Dict[str, Optional[Dict[str, Dict[str, Any]]]]
+        self, ext_product_types_conf: dict[str, Optional[dict[str, dict[str, Any]]]]
     ) -> None:
         """Update eodag product types list
 
@@ -815,7 +854,9 @@ class EODataAccessGateway:
                     ) or getattr(self.providers_config[provider], "api", None)
                     if search_plugin_config is None:
                         continue
-                    if not hasattr(search_plugin_config, "discover_product_types"):
+                    if not getattr(
+                        search_plugin_config, "discover_product_types", {}
+                    ).get("fetch_url", None):
                         # conf has been updated and provider product types are no more discoverable
                         continue
                     provider_products_config = (
@@ -827,7 +868,7 @@ class EODataAccessGateway:
                         provider,
                     )
                     continue
-                new_product_types: List[str] = []
+                new_product_types: list[str] = []
                 for (
                     new_product_type,
                     new_product_type_conf,
@@ -874,7 +915,7 @@ class EODataAccessGateway:
                             new_product_types.append(new_product_type)
                 if new_product_types:
                     logger.debug(
-                        f"Added product types {str(new_product_types)} for {provider}"
+                        f"Added {len(new_product_types)} product types for {provider}"
                     )
 
             elif provider not in self.providers_config:
@@ -890,7 +931,7 @@ class EODataAccessGateway:
 
     def available_providers(
         self, product_type: Optional[str] = None, by_group: bool = False
-    ) -> List[str]:
+    ) -> list[str]:
         """Gives the sorted list of the available providers or groups
 
         The providers or groups are sorted first by their priority level in descending order,
@@ -917,7 +958,7 @@ class EODataAccessGateway:
 
         # If by_group is True, keep only the highest priority for each group
         if by_group:
-            group_priority: Dict[str, int] = {}
+            group_priority: dict[str, int] = {}
             for name, priority in providers:
                 if name not in group_priority or priority > group_priority[name]:
                     group_priority[name] = priority
@@ -984,7 +1025,7 @@ class EODataAccessGateway:
         missionStartDate: Optional[str] = None,
         missionEndDate: Optional[str] = None,
         **kwargs: Any,
-    ) -> List[str]:
+    ) -> list[str]:
         """
         Find EODAG product type IDs that best match a set of search parameters.
 
@@ -1042,24 +1083,32 @@ class EODataAccessGateway:
             query = p.parse(text)
             results = searcher.search(query, limit=None)
 
-            guesses: List[Dict[str, str]] = [dict(r) for r in results or []]
+            guesses: list[dict[str, str]] = [dict(r) for r in results or []]
 
         # datetime filtering
         if missionStartDate or missionEndDate:
+            min_aware = datetime.datetime.min.replace(tzinfo=datetime.timezone.utc)
+            max_aware = datetime.datetime.max.replace(tzinfo=datetime.timezone.utc)
             guesses = [
                 g
                 for g in guesses
                 if (
-                    not missionEndDate
-                    or g.get("missionStartDate")
-                    and rfc3339_str_to_datetime(g["missionStartDate"])
-                    <= rfc3339_str_to_datetime(missionEndDate)
-                )
-                and (
-                    not missionStartDate
-                    or g.get("missionEndDate")
-                    and rfc3339_str_to_datetime(g["missionEndDate"])
-                    >= rfc3339_str_to_datetime(missionStartDate)
+                    max(
+                        rfc3339_str_to_datetime(missionStartDate)
+                        if missionStartDate
+                        else min_aware,
+                        rfc3339_str_to_datetime(g["missionStartDate"])
+                        if g.get("missionStartDate")
+                        else min_aware,
+                    )
+                    <= min(
+                        rfc3339_str_to_datetime(missionEndDate)
+                        if missionEndDate
+                        else max_aware,
+                        rfc3339_str_to_datetime(g["missionEndDate"])
+                        if g.get("missionEndDate")
+                        else max_aware,
+                    )
                 )
             ]
 
@@ -1075,8 +1124,8 @@ class EODataAccessGateway:
         raise_errors: bool = False,
         start: Optional[str] = None,
         end: Optional[str] = None,
-        geom: Optional[Union[str, Dict[str, float], BaseGeometry]] = None,
-        locations: Optional[Dict[str, str]] = None,
+        geom: Optional[Union[str, dict[str, float], BaseGeometry]] = None,
+        locations: Optional[dict[str, str]] = None,
         provider: Optional[str] = None,
         count: bool = False,
         **kwargs: Any,
@@ -1155,7 +1204,7 @@ class EODataAccessGateway:
             items_per_page=items_per_page,
         )
 
-        self.search_errors = set()
+        errors: list[tuple[str, Exception]] = []
         # Loop over available providers and return the first non-empty results
         for i, search_plugin in enumerate(search_plugins):
             search_plugin.clear()
@@ -1165,25 +1214,27 @@ class EODataAccessGateway:
                 raise_errors=raise_errors,
                 **search_kwargs,
             )
+            errors.extend(search_results.errors)
             if len(search_results) == 0 and i < len(search_plugins) - 1:
                 logger.warning(
                     f"No result could be obtained from provider {search_plugin.provider}, "
                     "we will try to get the data from another provider",
                 )
             elif len(search_results) > 0:
+                search_results.errors = errors
                 return search_results
 
         if i > 1:
             logger.error("No result could be obtained from any available provider")
-        return SearchResult([], 0) if count else SearchResult([])
+        return SearchResult([], 0, errors) if count else SearchResult([], errors=errors)
 
     def search_iter_page(
         self,
         items_per_page: int = DEFAULT_ITEMS_PER_PAGE,
         start: Optional[str] = None,
         end: Optional[str] = None,
-        geom: Optional[Union[str, Dict[str, float], BaseGeometry]] = None,
-        locations: Optional[Dict[str, str]] = None,
+        geom: Optional[Union[str, dict[str, float], BaseGeometry]] = None,
+        locations: Optional[dict[str, str]] = None,
         **kwargs: Any,
     ) -> Iterator[SearchResult]:
         """Iterate over the pages of a products search.
@@ -1359,8 +1410,8 @@ class EODataAccessGateway:
         items_per_page: Optional[int] = None,
         start: Optional[str] = None,
         end: Optional[str] = None,
-        geom: Optional[Union[str, Dict[str, float], BaseGeometry]] = None,
-        locations: Optional[Dict[str, str]] = None,
+        geom: Optional[Union[str, dict[str, float], BaseGeometry]] = None,
+        locations: Optional[dict[str, str]] = None,
         **kwargs: Any,
     ) -> SearchResult:
         """Search and return all the products matching the search criteria.
@@ -1435,7 +1486,7 @@ class EODataAccessGateway:
                 )
                 or DEFAULT_MAX_ITEMS_PER_PAGE
             )
-            logger.debug(
+            logger.info(
                 "Searching for all the products with provider %s and a maximum of %s "
                 "items per page.",
                 search_plugin.provider,
@@ -1503,7 +1554,7 @@ class EODataAccessGateway:
             try:
                 product_type = self.get_product_type_from_alias(product_type)
             except NoMatchingProductType:
-                logger.warning("product type %s not found", product_type)
+                logger.debug("product type %s not found", product_type)
         get_search_plugins_kwargs = dict(provider=provider, product_type=product_type)
         search_plugins = self._plugins_manager.get_search_plugins(
             **get_search_plugins_kwargs
@@ -1526,7 +1577,7 @@ class EODataAccessGateway:
                 "max_items_per_page", DEFAULT_MAX_ITEMS_PER_PAGE
             )
             kwargs.update(items_per_page=items_per_page)
-            if isinstance(plugin, BuildPostSearchResult):
+            if isinstance(plugin, PostJsonSearch):
                 kwargs.update(
                     items_per_page=items_per_page,
                     _dc_qs=_dc_qs,
@@ -1544,9 +1595,10 @@ class EODataAccessGateway:
                     **kwargs,
                 ):
                     results.data.extend(page_results.data)
-            except Exception:
+            except Exception as e:
                 if kwargs.get("raise_errors"):
                     raise
+                logger.warning(e)
                 continue
 
             # try using crunch to get unique result
@@ -1580,20 +1632,16 @@ class EODataAccessGateway:
         if not getattr(plugin.config, "discover_product_types", {}).get("fetch_url"):
             return None
 
-        kwargs: Dict[str, Any] = {"productType": product_type}
+        kwargs: dict[str, Any] = {"productType": product_type}
 
         # append auth if needed
         if getattr(plugin.config, "need_auth", False):
-            auth_plugin = self._plugins_manager.get_auth_plugin(plugin.provider)
-            if auth_plugin and callable(getattr(auth_plugin, "authenticate", None)):
-                try:
-                    kwargs["auth"] = auth_plugin.authenticate()
-                except (AuthenticationError, MisconfiguredError) as e:
-                    logger.warning(f"Could not authenticate on {provider}: {str(e)}")
-            else:
-                logger.warning(
-                    f"Could not authenticate on {provider} using {auth_plugin} plugin"
-                )
+            if auth := self._plugins_manager.get_auth(
+                plugin.provider,
+                getattr(plugin.config, "api_endpoint", None),
+                plugin.config,
+            ):
+                kwargs["auth"] = auth
 
         product_type_config = plugin.discover_product_types(**kwargs)
         self.update_product_types_list({provider: product_type_config})
@@ -1602,11 +1650,11 @@ class EODataAccessGateway:
         self,
         start: Optional[str] = None,
         end: Optional[str] = None,
-        geom: Optional[Union[str, Dict[str, float], BaseGeometry]] = None,
-        locations: Optional[Dict[str, str]] = None,
+        geom: Optional[Union[str, dict[str, float], BaseGeometry]] = None,
+        locations: Optional[dict[str, str]] = None,
         provider: Optional[str] = None,
         **kwargs: Any,
-    ) -> Tuple[List[Union[Search, Api]], Dict[str, Any]]:
+    ) -> tuple[list[Union[Search, Api]], dict[str, Any]]:
         """Internal method to prepare the search kwargs and get the search plugins.
 
         Product query:
@@ -1714,16 +1762,16 @@ class EODataAccessGateway:
 
         preferred_provider = self.get_preferred_provider()[0]
 
-        search_plugins: List[Union[Search, Api]] = []
+        search_plugins: list[Union[Search, Api]] = []
         for plugin in self._plugins_manager.get_search_plugins(
             product_type=product_type, provider=provider
         ):
-            # exclude BuildPostSearchResult plugins from search fallback for unknow product_type
+            # exclude MeteoblueSearch plugins from search fallback for unknown product_type
             if (
                 provider != plugin.provider
                 and preferred_provider != plugin.provider
                 and product_type not in self.product_types_config
-                and isinstance(plugin, BuildPostSearchResult)
+                and isinstance(plugin, MeteoblueSearch)
             ):
                 continue
             search_plugins.append(plugin)
@@ -1732,12 +1780,10 @@ class EODataAccessGateway:
             provider = preferred_provider
         providers = [plugin.provider for plugin in search_plugins]
         if provider not in providers:
-            logger.warning(
-                "Product type '%s' is not available with provider '%s'. "
-                "Searching it on provider '%s' instead.",
+            logger.debug(
+                "Product type '%s' is not available with preferred provider '%s'.",
                 product_type,
                 provider,
-                search_plugins[0].provider,
             )
         else:
             provider_plugin = list(
@@ -1745,35 +1791,10 @@ class EODataAccessGateway:
             )[0]
             search_plugins.remove(provider_plugin)
             search_plugins.insert(0, provider_plugin)
-            logger.info(
-                "Searching product type '%s' on provider: %s",
-                product_type,
-                search_plugins[0].provider,
-            )
         # Add product_types_config to plugin config. This dict contains product
         # type metadata that will also be stored in each product's properties.
         for search_plugin in search_plugins:
-            try:
-                search_plugin.config.product_type_config = dict(
-                    [
-                        p
-                        for p in self.list_product_types(
-                            search_plugin.provider, fetch_providers=False
-                        )
-                        if p["_id"] == product_type
-                    ][0],
-                    **{"productType": product_type},
-                )
-                # If the product isn't in the catalog, it's a generic product type.
-            except IndexError:
-                # Construct the GENERIC_PRODUCT_TYPE metadata
-                search_plugin.config.product_type_config = dict(
-                    ID=GENERIC_PRODUCT_TYPE,
-                    **self.product_types_config[GENERIC_PRODUCT_TYPE],
-                    productType=product_type,
-                )
-            # Remove the ID since this is equal to productType.
-            search_plugin.config.product_type_config.pop("ID", None)
+            self._attach_product_type_config(search_plugin, product_type)
 
         return search_plugins, kwargs
 
@@ -1793,6 +1814,7 @@ class EODataAccessGateway:
         :param kwargs: Some other criteria that will be used to do the search
         :returns: A collection of EO products matching the criteria
         """
+        logger.info("Searching on provider %s", search_plugin.provider)
         max_items_per_page = getattr(search_plugin.config, "pagination", {}).get(
             "max_items_per_page", DEFAULT_MAX_ITEMS_PER_PAGE
         )
@@ -1810,19 +1832,23 @@ class EODataAccessGateway:
                 max_items_per_page,
             )
 
-        need_auth = getattr(search_plugin.config, "need_auth", False)
-        auth_plugin = self._plugins_manager.get_auth_plugin(search_plugin.provider)
-        can_authenticate = callable(getattr(auth_plugin, "authenticate", None))
-
-        results: List[EOProduct] = []
+        results: list[EOProduct] = []
         total_results: Optional[int] = 0 if count else None
 
+        errors: list[tuple[str, Exception]] = []
+
         try:
             prep = PreparedSearch(count=count)
-            if need_auth and auth_plugin and can_authenticate:
-                prep.auth = auth_plugin.authenticate()
 
-            prep.auth_plugin = auth_plugin
+            # append auth if needed
+            if getattr(search_plugin.config, "need_auth", False):
+                if auth := self._plugins_manager.get_auth(
+                    search_plugin.provider,
+                    getattr(search_plugin.config, "api_endpoint", None),
+                    search_plugin.config,
+                ):
+                    prep.auth = auth
+
             prep.page = kwargs.pop("page", None)
             prep.items_per_page = kwargs.pop("items_per_page", None)
 
@@ -1876,12 +1902,31 @@ class EODataAccessGateway:
                             eo_product.product_type
                         )
                 except NoMatchingProductType:
-                    logger.warning("product type %s not found", eo_product.product_type)
+                    logger.debug("product type %s not found", eo_product.product_type)
 
                 if eo_product.search_intersection is not None:
                     download_plugin = self._plugins_manager.get_download_plugin(
                         eo_product
                     )
+                    if len(eo_product.assets) > 0:
+                        matching_url = next(iter(eo_product.assets.values()))["href"]
+                    elif eo_product.properties.get("storageStatus") != ONLINE_STATUS:
+                        matching_url = eo_product.properties.get(
+                            "orderLink"
+                        ) or eo_product.properties.get("downloadLink")
+                    else:
+                        matching_url = eo_product.properties.get("downloadLink")
+
+                    try:
+                        auth_plugin = next(
+                            self._plugins_manager.get_auth_plugins(
+                                search_plugin.provider,
+                                matching_url=matching_url,
+                                matching_conf=download_plugin.config,
+                            )
+                        )
+                    except StopIteration:
+                        auth_plugin = None
                     eo_product.register_downloader(download_plugin, auth_plugin)
 
             results.extend(res)
@@ -1911,13 +1956,6 @@ class EODataAccessGateway:
                         "the total number of products matching the search criteria"
                     )
         except Exception as e:
-            log_msg = f"No result from provider '{search_plugin.provider}' due to an error during search."
-            if not raise_errors:
-                log_msg += " Raise verbosity of log messages for details"
-            logger.info(log_msg)
-            # keep only the message from exception args
-            if len(e.args) > 1:
-                e.args = (e.args[0],)
             if raise_errors:
                 # Raise the error, letting the application wrapping eodag know that
                 # something went bad. This way it will be able to decide what to do next
@@ -1927,8 +1965,8 @@ class EODataAccessGateway:
                     "Error while searching on provider %s (ignored):",
                     search_plugin.provider,
                 )
-                self.search_errors.add((search_plugin.provider, e))
-        return SearchResult(results, total_results)
+                errors.append((search_plugin.provider, e))
+        return SearchResult(results, total_results, errors)
 
     def crunch(self, results: SearchResult, **kwargs: Any) -> SearchResult:
         """Apply the filters given through the keyword arguments to the results
@@ -1945,7 +1983,7 @@ class EODataAccessGateway:
         return results
 
     @staticmethod
-    def group_by_extent(searches: List[SearchResult]) -> List[SearchResult]:
+    def group_by_extent(searches: list[SearchResult]) -> list[SearchResult]:
         """Combines multiple SearchResults and return a list of SearchResults grouped
         by extent (i.e. bounding box).
 
@@ -1954,7 +1992,7 @@ class EODataAccessGateway:
         """
         # Dict with extents as keys, each extent being defined by a str
         # "{minx}{miny}{maxx}{maxy}" (each float rounded to 2 dec).
-        products_grouped_by_extent: Dict[str, Any] = {}
+        products_grouped_by_extent: dict[str, Any] = {}
 
         for search in searches:
             for product in search:
@@ -1973,16 +2011,16 @@ class EODataAccessGateway:
         search_result: SearchResult,
         downloaded_callback: Optional[DownloadedCallback] = None,
         progress_callback: Optional[ProgressCallback] = None,
-        wait: int = DEFAULT_DOWNLOAD_WAIT,
-        timeout: int = DEFAULT_DOWNLOAD_TIMEOUT,
+        wait: float = DEFAULT_DOWNLOAD_WAIT,
+        timeout: float = DEFAULT_DOWNLOAD_TIMEOUT,
         **kwargs: Unpack[DownloadConf],
-    ) -> List[str]:
+    ) -> list[str]:
         """Download all products resulting from a search.
 
         :param search_result: A collection of EO products resulting from a search
         :param downloaded_callback: (optional) A method or a callable object which takes
                                     as parameter the ``product``. You can use the base class
-                                    :class:`~eodag.api.product.DownloadedCallback` and override
+                                    :class:`~eodag.utils.DownloadedCallback` and override
                                     its ``__call__`` method. Will be called each time a product
                                     finishes downloading
         :param progress_callback: (optional) A method or a callable object
@@ -2061,12 +2099,12 @@ class EODataAccessGateway:
         products = self.deserialize(filename)
         for i, product in enumerate(products):
             if product.downloader is None:
+                downloader = self._plugins_manager.get_download_plugin(product)
                 auth = product.downloader_auth
                 if auth is None:
-                    auth = self._plugins_manager.get_auth_plugin(product.provider)
-                products[i].register_downloader(
-                    self._plugins_manager.get_download_plugin(product), auth
-                )
+                    auth = self._plugins_manager.get_auth_plugin(downloader, product)
+                products[i].register_downloader(downloader, auth)
+
         return products
 
     @_deprecated(
@@ -2090,7 +2128,7 @@ class EODataAccessGateway:
 
         :param filename: A filename containing features encoded as a geojson
         :param recursive: (optional) Browse recursively in child nodes if True
-        :param max_connections: (optional) Maximum number of connections for HTTP requests
+        :param max_connections: (optional) Maximum number of connections for concurrent HTTP requests
         :param provider: (optional) Data provider
         :param productType: (optional) Data product type
         :param timeout: (optional) Timeout in seconds for each internal HTTP request
@@ -2136,8 +2174,8 @@ class EODataAccessGateway:
         self,
         product: EOProduct,
         progress_callback: Optional[ProgressCallback] = None,
-        wait: int = DEFAULT_DOWNLOAD_WAIT,
-        timeout: int = DEFAULT_DOWNLOAD_TIMEOUT,
+        wait: float = DEFAULT_DOWNLOAD_WAIT,
+        timeout: float = DEFAULT_DOWNLOAD_TIMEOUT,
         **kwargs: Unpack[DownloadConf],
     ) -> str:
         """Download a single product.
@@ -2197,12 +2235,11 @@ class EODataAccessGateway:
 
     def _setup_downloader(self, product: EOProduct) -> None:
         if product.downloader is None:
+            downloader = self._plugins_manager.get_download_plugin(product)
             auth = product.downloader_auth
             if auth is None:
-                auth = self._plugins_manager.get_auth_plugin(product.provider)
-            product.register_downloader(
-                self._plugins_manager.get_download_plugin(product), auth
-            )
+                auth = self._plugins_manager.get_auth_plugin(downloader, product)
+            product.register_downloader(downloader, auth)
 
     def get_cruncher(self, name: str, **options: Any) -> Crunch:
         """Build a crunch plugin from a configuration
@@ -2216,79 +2253,115 @@ class EODataAccessGateway:
         return self._plugins_manager.get_crunch_plugin(name, **plugin_conf)
 
     def list_queryables(
-        self, provider: Optional[str] = None, **kwargs: Any
-    ) -> Dict[str, Annotated[Any, FieldInfo]]:
+        self,
+        provider: Optional[str] = None,
+        fetch_providers: bool = True,
+        **kwargs: Any,
+    ) -> QueryablesDict:
         """Fetch the queryable properties for a given product type and/or provider.
 
         :param provider: (optional) The provider.
+        :param fetch_providers: If new product types should be fetched from the providers; default: True
         :param kwargs: additional filters for queryables (`productType` or other search
                        arguments)
 
         :raises UnsupportedProductType: If the specified product type is not available for the
                                         provider.
 
-        :returns: A dict containing the EODAG queryable properties, associating
-                  parameters to their annotated type
+        :returns: A :class:`~eodag.api.product.queryables.QuerybalesDict` containing the EODAG queryable
+                  properties, associating parameters to their annotated type, and a additional_properties attribute
         """
-        available_product_types = [
+        # only fetch providers if product type is not found
+        available_product_types: list[str] = [
             pt["ID"]
             for pt in self.list_product_types(provider=provider, fetch_providers=False)
         ]
-        product_type = kwargs.get("productType")
+        product_type: Optional[str] = kwargs.get("productType")
+        pt_alias: Optional[str] = product_type
 
         if product_type:
+            if product_type not in available_product_types:
+                if fetch_providers:
+                    # fetch providers and try again
+                    available_product_types = [
+                        pt["ID"]
+                        for pt in self.list_product_types(
+                            provider=provider, fetch_providers=True
+                        )
+                    ]
+                raise UnsupportedProductType(f"{product_type} is not available.")
             try:
                 kwargs["productType"] = product_type = self.get_product_type_from_alias(
                     product_type
                 )
             except NoMatchingProductType as e:
-                raise UnsupportedProductType(f"{product_type} is not available") from e
-
-        if product_type and product_type not in available_product_types:
-            self.fetch_product_types_list()
+                raise UnsupportedProductType(f"{product_type} is not available.") from e
 
         if not provider and not product_type:
-            return model_fields_to_annotated(CommonQueryables.model_fields)
+            return QueryablesDict(
+                additional_properties=True,
+                **model_fields_to_annotated(CommonQueryables.model_fields),
+            )
 
-        providers_queryables: Dict[str, Dict[str, Annotated[Any, FieldInfo]]] = {}
+        additional_properties = False
+        additional_information = []
+        queryable_properties: dict[str, Any] = {}
 
         for plugin in self._plugins_manager.get_search_plugins(product_type, provider):
+            # attach product type config
+            product_type_configs: dict[str, Any] = {}
+            if product_type:
+                self._attach_product_type_config(plugin, product_type)
+                product_type_configs[product_type] = plugin.config.product_type_config
+            else:
+                for pt in available_product_types:
+                    self._attach_product_type_config(plugin, pt)
+                    product_type_configs[pt] = plugin.config.product_type_config
+
+            # authenticate if required
             if getattr(plugin.config, "need_auth", False) and (
-                auth := self._plugins_manager.get_auth_plugin(plugin.provider)
+                auth := self._plugins_manager.get_auth_plugin(plugin)
             ):
-                plugin.auth = auth.authenticate()
-            providers_queryables[plugin.provider] = plugin.list_queryables(
-                filters=kwargs, product_type=product_type
-            )
-
-        queryable_keys: Set[str] = set.intersection(  # type: ignore
-            *[set(q.keys()) for q in providers_queryables.values()]
-        )
-        queryables = {
-            k: v
-            for k, v in list(providers_queryables.values())[0].items()
-            if k in queryable_keys
-        }
+                try:
+                    plugin.auth = auth.authenticate()
+                except AuthenticationError:
+                    logger.debug(
+                        "queryables from provider %s could not be fetched due to an authentication error",
+                        plugin.provider,
+                    )
 
-        # always keep at least CommonQueryables
-        common_queryables = copy_deepcopy(CommonQueryables.model_fields)
-        for key, queryable in common_queryables.items():
-            if key in kwargs:
-                queryable.default = kwargs[key]
+            plugin_queryables = plugin.list_queryables(
+                kwargs,
+                available_product_types,
+                product_type_configs,
+                product_type,
+                pt_alias,
+            )
 
-        queryables.update(model_fields_to_annotated(common_queryables))
+            if plugin_queryables.additional_information:
+                additional_information.append(
+                    f"{plugin.provider}: {plugin_queryables.additional_information}"
+                )
+            queryable_properties = {**plugin_queryables, **queryable_properties}
+            additional_properties = (
+                additional_properties or plugin_queryables.additional_properties
+            )
 
-        return queryables
+        return QueryablesDict(
+            additional_properties=additional_properties,
+            additional_information=" | ".join(additional_information),
+            **queryable_properties,
+        )
 
-    def available_sortables(self) -> Dict[str, Optional[ProviderSortables]]:
+    def available_sortables(self) -> dict[str, Optional[ProviderSortables]]:
         """For each provider, gives its available sortable parameter(s) and its maximum
         number of them if it supports the sorting feature, otherwise gives None.
 
-        :returns: A dictionnary with providers as keys and dictionnary of sortable parameter(s) and
+        :returns: A dictionary with providers as keys and dictionary of sortable parameter(s) and
                   its (their) maximum number as value(s).
         :raises: :class:`~eodag.utils.exceptions.UnsupportedProvider`
         """
-        sortables: Dict[str, Optional[ProviderSortables]] = {}
+        sortables: dict[str, Optional[ProviderSortables]] = {}
         provider_search_plugins = self._plugins_manager.get_search_plugins()
         for provider_search_plugin in provider_search_plugins:
             provider = provider_search_plugin.provider
@@ -2311,3 +2384,30 @@ class EODataAccessGateway:
                 ],
             }
         return sortables
+
+    def _attach_product_type_config(self, plugin: Search, product_type: str) -> None:
+        """
+        Attach product_types_config to plugin config. This dict contains product
+        type metadata that will also be stored in each product's properties.
+        """
+        try:
+            plugin.config.product_type_config = dict(
+                [
+                    p
+                    for p in self.list_product_types(
+                        plugin.provider, fetch_providers=False
+                    )
+                    if p["_id"] == product_type
+                ][0],
+                **{"productType": product_type},
+            )
+            # If the product isn't in the catalog, it's a generic product type.
+        except IndexError:
+            # Construct the GENERIC_PRODUCT_TYPE metadata
+            plugin.config.product_type_config = dict(
+                ID=GENERIC_PRODUCT_TYPE,
+                **self.product_types_config[GENERIC_PRODUCT_TYPE],
+                productType=product_type,
+            )
+        # Remove the ID since this is equal to productType.
+        plugin.config.product_type_config.pop("ID", None)