eodag 3.0.0b3__py3-none-any.whl → 3.1.0b1__py3-none-any.whl

eodag/api/core.py

@@ -17,29 +17,34 @@
 # limitations under the License.
 from __future__ import annotations
 
+import datetime
 import logging
 import os
 import re
 import shutil
 import tempfile
 from operator import itemgetter
-from typing import TYPE_CHECKING, Any, Dict, Iterator, List, Optional, Set, Tuple, Union
+from typing import TYPE_CHECKING, Any, Dict, Iterator, List, Optional, Tuple, Union
 
 import geojson
 import pkg_resources
 import yaml.parser
 from pkg_resources import resource_filename
-from pydantic.fields import FieldInfo
 from whoosh import analysis, fields
 from whoosh.fields import Schema
-from whoosh.index import create_in, exists_in, open_dir
+from whoosh.index import exists_in, open_dir
 from whoosh.qparser import QueryParser
 
-from eodag.api.product.metadata_mapping import mtd_cfg_as_conversion_and_querypath
+from eodag.api.product.metadata_mapping import (
+    ONLINE_STATUS,
+    mtd_cfg_as_conversion_and_querypath,
+)
 from eodag.api.search_result import SearchResult
 from eodag.config import (
+    PLUGINS_TOPICS_KEYS,
     PluginConfig,
     SimpleYamlProxyConfig,
+    credentials_in_auth,
     get_ext_product_types_conf,
     load_default_config,
     load_stac_provider_config,
@@ -48,13 +53,15 @@ from eodag.config import (
     override_config_from_file,
     override_config_from_mapping,
     provider_config_init,
+    share_credentials,
 )
 from eodag.plugins.manager import PluginManager
 from eodag.plugins.search import PreparedSearch
-from eodag.plugins.search.build_search_result import BuildPostSearchResult
+from eodag.plugins.search.build_search_result import MeteoblueSearch
+from eodag.plugins.search.qssearch import PostJsonSearch
 from eodag.types import model_fields_to_annotated
-from eodag.types.queryables import CommonQueryables
-from eodag.types.whoosh import EODAGQueryParser
+from eodag.types.queryables import CommonQueryables, QueryablesDict
+from eodag.types.whoosh import EODAGQueryParser, create_in
 from eodag.utils import (
     DEFAULT_DOWNLOAD_TIMEOUT,
     DEFAULT_DOWNLOAD_WAIT,
@@ -65,7 +72,6 @@ from eodag.utils import (
     HTTP_REQ_TIMEOUT,
     MockResponse,
     _deprecated,
-    copy_deepcopy,
     get_geometry_from_various,
     makedirs,
     obj_md5sum,
@@ -76,7 +82,6 @@ from eodag.utils import (
 from eodag.utils.exceptions import (
     AuthenticationError,
     EodagError,
-    MisconfiguredError,
     NoMatchingProductType,
     PluginImplementationError,
     RequestError,
@@ -96,7 +101,7 @@ if TYPE_CHECKING:
     from eodag.plugins.search.base import Search
     from eodag.types import ProviderSortables
     from eodag.types.download_args import DownloadConf
-    from eodag.utils import Annotated, DownloadedCallback, ProgressCallback, Unpack
+    from eodag.utils import DownloadedCallback, ProgressCallback, Unpack
 
 logger = logging.getLogger("eodag.core")
 
@@ -166,10 +171,15 @@ class EODataAccessGateway:
         # Second level override: From environment variables
         override_config_from_env(self.providers_config)
 
+        # share credentials between updated plugins confs
+        share_credentials(self.providers_config)
+
         # init updated providers conf
-        stac_provider_config = load_stac_provider_config()
         for provider in self.providers_config.keys():
-            provider_config_init(self.providers_config[provider], stac_provider_config)
+            provider_config_init(
+                self.providers_config[provider],
+                load_stac_provider_config(),
+            )
 
         # re-build _plugins_manager using up-to-date providers_config
         self._plugins_manager.rebuild(self.providers_config)
@@ -197,9 +207,10 @@ class EODataAccessGateway:
                         "eodag",
                         os.path.join("resources", "locations_conf_template.yml"),
                     )
-                    with open(locations_conf_template) as infile, open(
-                        locations_conf_path, "w"
-                    ) as outfile:
+                    with (
+                        open(locations_conf_template) as infile,
+                        open(locations_conf_path, "w") as outfile,
+                    ):
                         # The template contains paths in the form of:
                         # /path/to/locations/file.shp
                         path_template = "/path/to/locations/"
@@ -215,7 +226,6 @@ class EODataAccessGateway:
                         os.path.join(self.conf_dir, "shp"),
                     )
         self.set_locations_conf(locations_conf_path)
-        self.search_errors: Set = set()
 
     def get_version(self) -> str:
         """Get eodag package version"""
@@ -296,13 +306,18 @@ class EODataAccessGateway:
                     product_type, **{"md5": self.product_types_config_md5}
                 )
                 # add to index
-                ix_writer.add_document(
-                    **{
-                        k: v
-                        for k, v in versioned_product_type.items()
-                        if k in product_types_schema.names()
-                    }
-                )
+                try:
+                    ix_writer.add_document(
+                        **{
+                            k: v
+                            for k, v in versioned_product_type.items()
+                            if k in product_types_schema.names()
+                        }
+                    )
+                except TypeError as e:
+                    logger.error(
+                        f"Cannot write product type {product_type['ID']} into index. e={e} product_type={product_type}"
+                    )
             ix_writer.commit()
 
     def set_preferred_provider(self, provider: str) -> None:
@@ -333,14 +348,24 @@ class EODataAccessGateway:
         preferred, priority = max(providers_with_priority, key=itemgetter(1))
         return preferred, priority
 
-    def update_providers_config(self, yaml_conf: str) -> None:
+    def update_providers_config(
+        self,
+        yaml_conf: Optional[str] = None,
+        dict_conf: Optional[Dict[str, Any]] = None,
+    ) -> None:
         """Update providers configuration with given input.
         Can be used to add a provider to existing configuration or update
         an existing one.
 
         :param yaml_conf: YAML formated provider configuration
+        :param dict_conf: provider configuration as dictionary in place of ``yaml_conf``
         """
-        conf_update = yaml.safe_load(yaml_conf)
+        if dict_conf is not None:
+            conf_update = dict_conf
+        elif yaml_conf is not None:
+            conf_update = yaml.safe_load(yaml_conf)
+        else:
+            return None
 
         # restore the pruned configuration
         for provider in list(self._pruned_providers_config.keys()):
@@ -355,9 +380,14 @@ class EODataAccessGateway:
 
         override_config_from_mapping(self.providers_config, conf_update)
 
-        stac_provider_config = load_stac_provider_config()
+        # share credentials between updated plugins confs
+        share_credentials(self.providers_config)
+
         for provider in conf_update.keys():
-            provider_config_init(self.providers_config[provider], stac_provider_config)
+            provider_config_init(
+                self.providers_config[provider],
+                load_stac_provider_config(),
+            )
             setattr(self.providers_config[provider], "product_types_fetched", False)
         # re-create _plugins_manager using up-to-date providers_config
         self._plugins_manager.build_product_type_to_provider_config_map()
@@ -419,14 +449,11 @@ class EODataAccessGateway:
 
         # api plugin usage: remove unneeded search/download/auth plugin conf
         if conf_dict[name].get("api"):
-            conf_dict[name].pop("search", None)
-            conf_dict[name].pop("download", None)
-            conf_dict[name].pop("auth", None)
+            for k in PLUGINS_TOPICS_KEYS:
+                if k != "api":
+                    conf_dict[name].pop(k, None)
 
-        override_config_from_mapping(self.providers_config, conf_dict)
-        provider_config_init(self.providers_config[name], load_stac_provider_config())
-        setattr(self.providers_config[name], "product_types_fetched", False)
-        self._plugins_manager.build_product_type_to_provider_config_map()
+        self.update_providers_config(dict_conf=conf_dict)
 
         if priority is None:
             self.set_preferred_provider(name)
@@ -452,12 +479,7 @@ class EODataAccessGateway:
 
             # check authentication
             if hasattr(conf, "api") and getattr(conf.api, "need_auth", False):
-                credentials_exist = any(
-                    [
-                        cred is not None
-                        for cred in getattr(conf.api, "credentials", {}).values()
-                    ]
-                )
+                credentials_exist = credentials_in_auth(conf.api)
                 if not credentials_exist:
                     # credentials needed but not found
                     self._pruned_providers_config[provider] = self.providers_config.pop(
@@ -469,7 +491,7 @@ class EODataAccessGateway:
                         provider,
                     )
             elif hasattr(conf, "search") and getattr(conf.search, "need_auth", False):
-                if not hasattr(conf, "auth"):
+                if not hasattr(conf, "auth") and not hasattr(conf, "search_auth"):
                     # credentials needed but no auth plugin was found
                     self._pruned_providers_config[provider] = self.providers_config.pop(
                         provider
@@ -480,11 +502,13 @@ class EODataAccessGateway:
                         provider,
                     )
                     continue
-                credentials_exist = any(
-                    [
-                        cred is not None
-                        for cred in getattr(conf.auth, "credentials", {}).values()
-                    ]
+                credentials_exist = (
+                    hasattr(conf, "search_auth")
+                    and credentials_in_auth(conf.search_auth)
+                ) or (
+                    not hasattr(conf, "search_auth")
+                    and hasattr(conf, "auth")
+                    and credentials_in_auth(conf.auth)
                 )
                 if not credentials_exist:
                     # credentials needed but not found
@@ -599,21 +623,32 @@ class EODataAccessGateway:
     def fetch_product_types_list(self, provider: Optional[str] = None) -> None:
         """Fetch product types list and update if needed
 
-        :param provider: (optional) The name of a provider for which product types list
-                         should be updated. Defaults to all providers (None value).
+        :param provider: The name of a provider or provider-group for which product types
+                         list should be updated. Defaults to all providers (None value).
         """
+        providers_to_fetch = list(self.providers_config.keys())
+        # check if some providers are grouped under a group name which is not a provider name
         if provider is not None and provider not in self.providers_config:
-            return
+            providers_to_fetch = [
+                p
+                for p, pconf in self.providers_config.items()
+                if provider == getattr(pconf, "group", None)
+            ]
+            if providers_to_fetch:
+                logger.info(
+                    f"Fetch product types for {provider} group: {', '.join(providers_to_fetch)}"
+                )
+            else:
+                return None
+        elif provider is not None:
+            providers_to_fetch = [provider]
 
         # providers discovery confs that are fetchable
         providers_discovery_configs_fetchable: Dict[str, Any] = {}
         # check if any provider has not already been fetched for product types
         already_fetched = True
-        for provider_to_fetch, provider_config in (
-            {provider: self.providers_config[provider]}.items()
-            if provider
-            else self.providers_config.items()
-        ):
+        for provider_to_fetch in providers_to_fetch:
+            provider_config = self.providers_config[provider_to_fetch]
             # get discovery conf
             if hasattr(provider_config, "search"):
                 provider_search_config = provider_config.search
@@ -735,11 +770,20 @@ class EODataAccessGateway:
     ) -> Optional[Dict[str, Any]]:
         """Fetch providers for product types
 
-        :param provider: (optional) The name of a provider to fetch. Defaults to all
-                         providers (None value).
+        :param provider: The name of a provider or provider-group to fetch. Defaults to
+                         all providers (None value).
         :returns: external product types configuration
         """
-        if provider and provider not in self.providers_config:
+        grouped_providers = [
+            p
+            for p, provider_config in self.providers_config.items()
+            if provider == getattr(provider_config, "group", None)
+        ]
+        if provider and provider not in self.providers_config and grouped_providers:
+            logger.info(
+                f"Discover product types for {provider} group: {', '.join(grouped_providers)}"
+            )
+        elif provider and provider not in self.providers_config:
             raise UnsupportedProvider(
                 f"The requested provider is not (yet) supported: {provider}"
             )
@@ -748,7 +792,9 @@ class EODataAccessGateway:
             p
             for p in (
                 [
-                    provider,
+                    p
+                    for p in self.providers_config
+                    if p in grouped_providers + [provider]
                 ]
                 if provider
                 else self.available_providers()
@@ -762,7 +808,9 @@ class EODataAccessGateway:
                 search_plugin_config = self.providers_config[provider].api
             else:
                 return None
-            if getattr(search_plugin_config, "discover_product_types", None):
+            if getattr(search_plugin_config, "discover_product_types", {}).get(
+                "fetch_url", None
+            ):
                 search_plugin: Union[Search, Api] = next(
                     self._plugins_manager.get_search_plugins(provider=provider)
                 )
@@ -773,23 +821,15 @@ class EODataAccessGateway:
                     continue
                 # append auth to search plugin if needed
                 if getattr(search_plugin.config, "need_auth", False):
-                    auth_plugin = self._plugins_manager.get_auth_plugin(
-                        search_plugin.provider
-                    )
-                    if auth_plugin and callable(
-                        getattr(auth_plugin, "authenticate", None)
+                    if auth := self._plugins_manager.get_auth(
+                        search_plugin.provider,
+                        getattr(search_plugin.config, "api_endpoint", None),
+                        search_plugin.config,
                     ):
-                        try:
-                            kwargs["auth"] = auth_plugin.authenticate()
-                        except (AuthenticationError, MisconfiguredError) as e:
-                            logger.warning(
-                                f"Could not authenticate on {provider}: {str(e)}"
-                            )
-                            ext_product_types_conf[provider] = None
-                            continue
+                        kwargs["auth"] = auth
                     else:
-                        logger.warning(
-                            f"Could not authenticate on {provider} using {auth_plugin} plugin"
+                        logger.debug(
+                            f"Could not authenticate on {provider} for product types discovery"
                         )
                         ext_product_types_conf[provider] = None
                         continue
@@ -815,7 +855,9 @@ class EODataAccessGateway:
                     ) or getattr(self.providers_config[provider], "api", None)
                     if search_plugin_config is None:
                         continue
-                    if not hasattr(search_plugin_config, "discover_product_types"):
+                    if not getattr(
+                        search_plugin_config, "discover_product_types", {}
+                    ).get("fetch_url", None):
                         # conf has been updated and provider product types are no more discoverable
                         continue
                     provider_products_config = (
@@ -874,7 +916,7 @@ class EODataAccessGateway:
                             new_product_types.append(new_product_type)
                 if new_product_types:
                     logger.debug(
-                        f"Added product types {str(new_product_types)} for {provider}"
+                        f"Added {len(new_product_types)} product types for {provider}"
                     )
 
             elif provider not in self.providers_config:
@@ -1046,20 +1088,28 @@ class EODataAccessGateway:
 
         # datetime filtering
         if missionStartDate or missionEndDate:
+            min_aware = datetime.datetime.min.replace(tzinfo=datetime.timezone.utc)
+            max_aware = datetime.datetime.max.replace(tzinfo=datetime.timezone.utc)
             guesses = [
                 g
                 for g in guesses
                 if (
-                    not missionEndDate
-                    or g.get("missionStartDate")
-                    and rfc3339_str_to_datetime(g["missionStartDate"])
-                    <= rfc3339_str_to_datetime(missionEndDate)
-                )
-                and (
-                    not missionStartDate
-                    or g.get("missionEndDate")
-                    and rfc3339_str_to_datetime(g["missionEndDate"])
-                    >= rfc3339_str_to_datetime(missionStartDate)
+                    max(
+                        rfc3339_str_to_datetime(missionStartDate)
+                        if missionStartDate
+                        else min_aware,
+                        rfc3339_str_to_datetime(g["missionStartDate"])
+                        if g.get("missionStartDate")
+                        else min_aware,
+                    )
+                    <= min(
+                        rfc3339_str_to_datetime(missionEndDate)
+                        if missionEndDate
+                        else max_aware,
+                        rfc3339_str_to_datetime(g["missionEndDate"])
+                        if g.get("missionEndDate")
+                        else max_aware,
+                    )
                 )
             ]
 
@@ -1155,7 +1205,7 @@ class EODataAccessGateway:
             items_per_page=items_per_page,
         )
 
-        self.search_errors = set()
+        errors: List[Tuple[str, Exception]] = []
         # Loop over available providers and return the first non-empty results
         for i, search_plugin in enumerate(search_plugins):
             search_plugin.clear()
@@ -1165,17 +1215,19 @@ class EODataAccessGateway:
                 raise_errors=raise_errors,
                 **search_kwargs,
             )
+            errors.extend(search_results.errors)
             if len(search_results) == 0 and i < len(search_plugins) - 1:
                 logger.warning(
                     f"No result could be obtained from provider {search_plugin.provider}, "
                     "we will try to get the data from another provider",
                 )
             elif len(search_results) > 0:
+                search_results.errors = errors
                 return search_results
 
         if i > 1:
             logger.error("No result could be obtained from any available provider")
-        return SearchResult([], 0) if count else SearchResult([])
+        return SearchResult([], 0, errors) if count else SearchResult([], errors=errors)
 
     def search_iter_page(
         self,
@@ -1435,7 +1487,7 @@ class EODataAccessGateway:
                 )
                 or DEFAULT_MAX_ITEMS_PER_PAGE
             )
-            logger.debug(
+            logger.info(
                 "Searching for all the products with provider %s and a maximum of %s "
                 "items per page.",
                 search_plugin.provider,
@@ -1503,7 +1555,7 @@ class EODataAccessGateway:
             try:
                 product_type = self.get_product_type_from_alias(product_type)
             except NoMatchingProductType:
-                logger.warning("product type %s not found", product_type)
+                logger.debug("product type %s not found", product_type)
         get_search_plugins_kwargs = dict(provider=provider, product_type=product_type)
         search_plugins = self._plugins_manager.get_search_plugins(
             **get_search_plugins_kwargs
@@ -1526,7 +1578,7 @@ class EODataAccessGateway:
                 "max_items_per_page", DEFAULT_MAX_ITEMS_PER_PAGE
             )
             kwargs.update(items_per_page=items_per_page)
-            if isinstance(plugin, BuildPostSearchResult):
+            if isinstance(plugin, PostJsonSearch):
                 kwargs.update(
                     items_per_page=items_per_page,
                     _dc_qs=_dc_qs,
@@ -1544,9 +1596,10 @@ class EODataAccessGateway:
                     **kwargs,
                 ):
                     results.data.extend(page_results.data)
-            except Exception:
+            except Exception as e:
                 if kwargs.get("raise_errors"):
                     raise
+                logger.warning(e)
                 continue
 
             # try using crunch to get unique result
@@ -1584,16 +1637,12 @@ class EODataAccessGateway:
 
         # append auth if needed
         if getattr(plugin.config, "need_auth", False):
-            auth_plugin = self._plugins_manager.get_auth_plugin(plugin.provider)
-            if auth_plugin and callable(getattr(auth_plugin, "authenticate", None)):
-                try:
-                    kwargs["auth"] = auth_plugin.authenticate()
-                except (AuthenticationError, MisconfiguredError) as e:
-                    logger.warning(f"Could not authenticate on {provider}: {str(e)}")
-            else:
-                logger.warning(
-                    f"Could not authenticate on {provider} using {auth_plugin} plugin"
-                )
+            if auth := self._plugins_manager.get_auth(
+                plugin.provider,
+                getattr(plugin.config, "api_endpoint", None),
+                plugin.config,
+            ):
+                kwargs["auth"] = auth
 
         product_type_config = plugin.discover_product_types(**kwargs)
         self.update_product_types_list({provider: product_type_config})
@@ -1718,12 +1767,12 @@ class EODataAccessGateway:
         for plugin in self._plugins_manager.get_search_plugins(
             product_type=product_type, provider=provider
         ):
-            # exclude BuildPostSearchResult plugins from search fallback for unknow product_type
+            # exclude MeteoblueSearch plugins from search fallback for unknown product_type
             if (
                 provider != plugin.provider
                 and preferred_provider != plugin.provider
                 and product_type not in self.product_types_config
-                and isinstance(plugin, BuildPostSearchResult)
+                and isinstance(plugin, MeteoblueSearch)
             ):
                 continue
             search_plugins.append(plugin)
@@ -1732,12 +1781,10 @@ class EODataAccessGateway:
             provider = preferred_provider
         providers = [plugin.provider for plugin in search_plugins]
         if provider not in providers:
-            logger.warning(
-                "Product type '%s' is not available with provider '%s'. "
-                "Searching it on provider '%s' instead.",
+            logger.debug(
+                "Product type '%s' is not available with preferred provider '%s'.",
                 product_type,
                 provider,
-                search_plugins[0].provider,
             )
         else:
             provider_plugin = list(
@@ -1745,35 +1792,10 @@ class EODataAccessGateway:
             )[0]
             search_plugins.remove(provider_plugin)
             search_plugins.insert(0, provider_plugin)
-            logger.info(
-                "Searching product type '%s' on provider: %s",
-                product_type,
-                search_plugins[0].provider,
-            )
         # Add product_types_config to plugin config. This dict contains product
         # type metadata that will also be stored in each product's properties.
         for search_plugin in search_plugins:
-            try:
-                search_plugin.config.product_type_config = dict(
-                    [
-                        p
-                        for p in self.list_product_types(
-                            search_plugin.provider, fetch_providers=False
-                        )
-                        if p["_id"] == product_type
-                    ][0],
-                    **{"productType": product_type},
-                )
-                # If the product isn't in the catalog, it's a generic product type.
-            except IndexError:
-                # Construct the GENERIC_PRODUCT_TYPE metadata
-                search_plugin.config.product_type_config = dict(
-                    ID=GENERIC_PRODUCT_TYPE,
-                    **self.product_types_config[GENERIC_PRODUCT_TYPE],
-                    productType=product_type,
-                )
-            # Remove the ID since this is equal to productType.
-            search_plugin.config.product_type_config.pop("ID", None)
+            self._attach_product_type_config(search_plugin, product_type)
 
         return search_plugins, kwargs
 
@@ -1793,6 +1815,7 @@ class EODataAccessGateway:
         :param kwargs: Some other criteria that will be used to do the search
         :returns: A collection of EO products matching the criteria
         """
+        logger.info("Searching on provider %s", search_plugin.provider)
         max_items_per_page = getattr(search_plugin.config, "pagination", {}).get(
             "max_items_per_page", DEFAULT_MAX_ITEMS_PER_PAGE
         )
@@ -1810,19 +1833,23 @@ class EODataAccessGateway:
                 max_items_per_page,
             )
 
-        need_auth = getattr(search_plugin.config, "need_auth", False)
-        auth_plugin = self._plugins_manager.get_auth_plugin(search_plugin.provider)
-        can_authenticate = callable(getattr(auth_plugin, "authenticate", None))
-
         results: List[EOProduct] = []
         total_results: Optional[int] = 0 if count else None
 
+        errors: List[Tuple[str, Exception]] = []
+
         try:
             prep = PreparedSearch(count=count)
-            if need_auth and auth_plugin and can_authenticate:
-                prep.auth = auth_plugin.authenticate()
 
-            prep.auth_plugin = auth_plugin
+            # append auth if needed
+            if getattr(search_plugin.config, "need_auth", False):
+                if auth := self._plugins_manager.get_auth(
+                    search_plugin.provider,
+                    getattr(search_plugin.config, "api_endpoint", None),
+                    search_plugin.config,
+                ):
+                    prep.auth = auth
+
             prep.page = kwargs.pop("page", None)
             prep.items_per_page = kwargs.pop("items_per_page", None)
 
@@ -1876,12 +1903,31 @@ class EODataAccessGateway:
                             eo_product.product_type
                         )
                 except NoMatchingProductType:
-                    logger.warning("product type %s not found", eo_product.product_type)
+                    logger.debug("product type %s not found", eo_product.product_type)
 
                 if eo_product.search_intersection is not None:
                     download_plugin = self._plugins_manager.get_download_plugin(
                         eo_product
                     )
+                    if len(eo_product.assets) > 0:
+                        matching_url = next(iter(eo_product.assets.values()))["href"]
+                    elif eo_product.properties.get("storageStatus") != ONLINE_STATUS:
+                        matching_url = eo_product.properties.get(
+                            "orderLink"
+                        ) or eo_product.properties.get("downloadLink")
+                    else:
+                        matching_url = eo_product.properties.get("downloadLink")
+
+                    try:
+                        auth_plugin = next(
+                            self._plugins_manager.get_auth_plugins(
+                                search_plugin.provider,
+                                matching_url=matching_url,
+                                matching_conf=download_plugin.config,
+                            )
+                        )
+                    except StopIteration:
+                        auth_plugin = None
                     eo_product.register_downloader(download_plugin, auth_plugin)
 
             results.extend(res)
@@ -1911,13 +1957,6 @@ class EODataAccessGateway:
                         "the total number of products matching the search criteria"
                     )
         except Exception as e:
-            log_msg = f"No result from provider '{search_plugin.provider}' due to an error during search."
-            if not raise_errors:
-                log_msg += " Raise verbosity of log messages for details"
-            logger.info(log_msg)
-            # keep only the message from exception args
-            if len(e.args) > 1:
-                e.args = (e.args[0],)
             if raise_errors:
                 # Raise the error, letting the application wrapping eodag know that
                 # something went bad. This way it will be able to decide what to do next
@@ -1927,8 +1966,8 @@ class EODataAccessGateway:
                     "Error while searching on provider %s (ignored):",
                     search_plugin.provider,
                 )
-                self.search_errors.add((search_plugin.provider, e))
-        return SearchResult(results, total_results)
+                errors.append((search_plugin.provider, e))
+        return SearchResult(results, total_results, errors)
 
     def crunch(self, results: SearchResult, **kwargs: Any) -> SearchResult:
         """Apply the filters given through the keyword arguments to the results
@@ -1973,8 +2012,8 @@ class EODataAccessGateway:
         search_result: SearchResult,
         downloaded_callback: Optional[DownloadedCallback] = None,
         progress_callback: Optional[ProgressCallback] = None,
-        wait: int = DEFAULT_DOWNLOAD_WAIT,
-        timeout: int = DEFAULT_DOWNLOAD_TIMEOUT,
+        wait: float = DEFAULT_DOWNLOAD_WAIT,
+        timeout: float = DEFAULT_DOWNLOAD_TIMEOUT,
         **kwargs: Unpack[DownloadConf],
     ) -> List[str]:
         """Download all products resulting from a search.
@@ -1982,7 +2021,7 @@ class EODataAccessGateway:
         :param search_result: A collection of EO products resulting from a search
         :param downloaded_callback: (optional) A method or a callable object which takes
                                     as parameter the ``product``. You can use the base class
-                                    :class:`~eodag.api.product.DownloadedCallback` and override
+                                    :class:`~eodag.utils.DownloadedCallback` and override
                                     its ``__call__`` method. Will be called each time a product
                                     finishes downloading
         :param progress_callback: (optional) A method or a callable object
@@ -2061,12 +2100,12 @@ class EODataAccessGateway:
         products = self.deserialize(filename)
         for i, product in enumerate(products):
             if product.downloader is None:
+                downloader = self._plugins_manager.get_download_plugin(product)
                 auth = product.downloader_auth
                 if auth is None:
-                    auth = self._plugins_manager.get_auth_plugin(product.provider)
-                products[i].register_downloader(
-                    self._plugins_manager.get_download_plugin(product), auth
-                )
+                    auth = self._plugins_manager.get_auth_plugin(downloader, product)
+                products[i].register_downloader(downloader, auth)
+
         return products
 
     @_deprecated(
@@ -2090,7 +2129,7 @@ class EODataAccessGateway:
 
         :param filename: A filename containing features encoded as a geojson
         :param recursive: (optional) Browse recursively in child nodes if True
-        :param max_connections: (optional) Maximum number of connections for HTTP requests
+        :param max_connections: (optional) Maximum number of connections for concurrent HTTP requests
         :param provider: (optional) Data provider
         :param productType: (optional) Data product type
         :param timeout: (optional) Timeout in seconds for each internal HTTP request
@@ -2136,8 +2175,8 @@ class EODataAccessGateway:
         self,
         product: EOProduct,
         progress_callback: Optional[ProgressCallback] = None,
-        wait: int = DEFAULT_DOWNLOAD_WAIT,
-        timeout: int = DEFAULT_DOWNLOAD_TIMEOUT,
+        wait: float = DEFAULT_DOWNLOAD_WAIT,
+        timeout: float = DEFAULT_DOWNLOAD_TIMEOUT,
         **kwargs: Unpack[DownloadConf],
     ) -> str:
         """Download a single product.
@@ -2197,12 +2236,11 @@ class EODataAccessGateway:
 
     def _setup_downloader(self, product: EOProduct) -> None:
         if product.downloader is None:
+            downloader = self._plugins_manager.get_download_plugin(product)
             auth = product.downloader_auth
             if auth is None:
-                auth = self._plugins_manager.get_auth_plugin(product.provider)
-            product.register_downloader(
-                self._plugins_manager.get_download_plugin(product), auth
-            )
+                auth = self._plugins_manager.get_auth_plugin(downloader, product)
+            product.register_downloader(downloader, auth)
 
     def get_cruncher(self, name: str, **options: Any) -> Crunch:
         """Build a crunch plugin from a configuration
@@ -2216,67 +2254,96 @@ class EODataAccessGateway:
         return self._plugins_manager.get_crunch_plugin(name, **plugin_conf)
 
     def list_queryables(
-        self, provider: Optional[str] = None, **kwargs: Any
-    ) -> Dict[str, Annotated[Any, FieldInfo]]:
+        self,
+        provider: Optional[str] = None,
+        fetch_providers: bool = True,
+        **kwargs: Any,
+    ) -> QueryablesDict:
         """Fetch the queryable properties for a given product type and/or provider.
 
         :param provider: (optional) The provider.
+        :param fetch_providers: If new product types should be fetched from the providers; default: True
         :param kwargs: additional filters for queryables (`productType` or other search
                        arguments)
 
         :raises UnsupportedProductType: If the specified product type is not available for the
                                         provider.
 
-        :returns: A dict containing the EODAG queryable properties, associating
-                  parameters to their annotated type
+        :returns: A :class:`~eodag.api.product.queryables.QuerybalesDict` containing the EODAG queryable
+                  properties, associating parameters to their annotated type, and a additional_properties attribute
         """
+        # only fetch providers if product type is not found
         available_product_types = [
             pt["ID"]
             for pt in self.list_product_types(provider=provider, fetch_providers=False)
         ]
-        product_type = kwargs.get("productType")
+        product_type: Optional[str] = kwargs.get("productType")
+        pt_alias: Optional[str] = product_type
 
         if product_type:
+            if product_type not in available_product_types:
+                if fetch_providers:
+                    # fetch providers and try again
+                    available_product_types = [
+                        pt["ID"]
+                        for pt in self.list_product_types(
+                            provider=provider, fetch_providers=True
+                        )
+                    ]
+                raise UnsupportedProductType(f"{product_type} is not available.")
             try:
                 kwargs["productType"] = product_type = self.get_product_type_from_alias(
                     product_type
                 )
             except NoMatchingProductType as e:
-                raise UnsupportedProductType(f"{product_type} is not available") from e
-
-        if product_type and product_type not in available_product_types:
-            self.fetch_product_types_list()
+                raise UnsupportedProductType(f"{product_type} is not available.") from e
 
         if not provider and not product_type:
-            return model_fields_to_annotated(CommonQueryables.model_fields)
+            return QueryablesDict(
+                additional_properties=True,
+                **model_fields_to_annotated(CommonQueryables.model_fields),
+            )
 
-        providers_queryables: Dict[str, Dict[str, Annotated[Any, FieldInfo]]] = {}
+        queryables: QueryablesDict = QueryablesDict(
+            additional_properties=True, additional_information="", **{}
+        )
 
         for plugin in self._plugins_manager.get_search_plugins(product_type, provider):
+            # attach product type config
+            product_type_configs = {}
+            if product_type:
+                self._attach_product_type_config(plugin, product_type)
+                product_type_configs[product_type] = plugin.config.product_type_config
+            else:
+                for pt in available_product_types:
+                    self._attach_product_type_config(plugin, pt)
+                    product_type_configs[pt] = plugin.config.product_type_config
+
+            # authenticate if required
             if getattr(plugin.config, "need_auth", False) and (
-                auth := self._plugins_manager.get_auth_plugin(plugin.provider)
+                auth := self._plugins_manager.get_auth_plugin(plugin)
             ):
-                plugin.auth = auth.authenticate()
-            providers_queryables[plugin.provider] = plugin.list_queryables(
-                filters=kwargs, product_type=product_type
+                try:
+                    plugin.auth = auth.authenticate()
+                except AuthenticationError:
+                    logger.debug(
+                        "queryables from provider %s could not be fetched due to an authentication error",
+                        plugin.provider,
+                    )
+            plugin_queryables = plugin.list_queryables(
+                kwargs,
+                available_product_types,
+                product_type_configs,
+                product_type,
+                pt_alias,
             )
-
-        queryable_keys: Set[str] = set.intersection(  # type: ignore
-            *[set(q.keys()) for q in providers_queryables.values()]
-        )
-        queryables = {
-            k: v
-            for k, v in list(providers_queryables.values())[0].items()
-            if k in queryable_keys
-        }
-
-        # always keep at least CommonQueryables
-        common_queryables = copy_deepcopy(CommonQueryables.model_fields)
-        for key, queryable in common_queryables.items():
-            if key in kwargs:
-                queryable.default = kwargs[key]
-
-        queryables.update(model_fields_to_annotated(common_queryables))
+            queryables.update(plugin_queryables)
+            if not plugin_queryables.additional_properties:
+                queryables.additional_properties = False
+            if plugin_queryables.additional_information:
+                queryables.additional_information += (
+                    f"{provider}: {plugin_queryables.additional_information}"
+                )
 
         return queryables
 
@@ -2284,7 +2351,7 @@ class EODataAccessGateway:
         """For each provider, gives its available sortable parameter(s) and its maximum
         number of them if it supports the sorting feature, otherwise gives None.
 
-        :returns: A dictionnary with providers as keys and dictionnary of sortable parameter(s) and
+        :returns: A dictionary with providers as keys and dictionary of sortable parameter(s) and
                   its (their) maximum number as value(s).
         :raises: :class:`~eodag.utils.exceptions.UnsupportedProvider`
         """
@@ -2311,3 +2378,30 @@ class EODataAccessGateway:
                 ],
             }
         return sortables
+
+    def _attach_product_type_config(self, plugin: Search, product_type: str) -> None:
+        """
+        Attach product_types_config to plugin config. This dict contains product
+        type metadata that will also be stored in each product's properties.
+        """
+        try:
+            plugin.config.product_type_config = dict(
+                [
+                    p
+                    for p in self.list_product_types(
+                        plugin.provider, fetch_providers=False
+                    )
+                    if p["_id"] == product_type
+                ][0],
+                **{"productType": product_type},
+            )
+            # If the product isn't in the catalog, it's a generic product type.
+        except IndexError:
+            # Construct the GENERIC_PRODUCT_TYPE metadata
+            plugin.config.product_type_config = dict(
+                ID=GENERIC_PRODUCT_TYPE,
+                **self.product_types_config[GENERIC_PRODUCT_TYPE],
+                productType=product_type,
+            )
+        # Remove the ID since this is equal to productType.
+        plugin.config.product_type_config.pop("ID", None)