eodag 3.2.1__py3-none-any.whl → 3.3.1__py3-none-any.whl

eodag/api/core.py

@@ -1599,6 +1599,7 @@ class EODataAccessGateway:
                 if kwargs.get("raise_errors"):
                     raise
                 logger.warning(e)
+                results.errors.append((plugin.provider, e))
                 continue
 
             # try using crunch to get unique result
@@ -1622,7 +1623,7 @@ class EODataAccessGateway:
                     "Several products found for this id (%s). You may try searching using more selective criteria.",
                     results,
                 )
-        return SearchResult([], 0)
+        return SearchResult([], 0, results.errors)
 
     def _fetch_external_product_type(self, provider: str, product_type: str):
         plugins = self._plugins_manager.get_search_plugins(provider=provider)

eodag/api/search_result.py

@@ -36,7 +36,7 @@ if TYPE_CHECKING:
     from eodag.plugins.crunch.base import Crunch
 
 
-class SearchResult(UserList):
+class SearchResult(UserList[EOProduct]):
     """An object representing a collection of :class:`~eodag.api.product._product.EOProduct` resulting from a search.
 
     :param products: A list of products resulting from a search
@@ -46,8 +46,6 @@ class SearchResult(UserList):
     :ivar number_matched: Estimated total number of matching results
     """
 
-    data: list[EOProduct]
-
     errors: Annotated[
         list[tuple[str, Exception]], Doc("Tuple of provider name, exception")
     ]
@@ -56,11 +54,11 @@ class SearchResult(UserList):
         self,
         products: list[EOProduct],
         number_matched: Optional[int] = None,
-        errors: list[tuple[str, Exception]] = [],
+        errors: Optional[list[tuple[str, Exception]]] = None,
     ) -> None:
         super().__init__(products)
         self.number_matched = number_matched
-        self.errors = errors
+        self.errors = errors if errors is not None else []
 
     def crunch(self, cruncher: Crunch, **search_params: Any) -> SearchResult:
         """Do some crunching with the underlying EO products.
@@ -193,7 +191,7 @@ class SearchResult(UserList):
                 <details><summary style='color: grey; font-family: monospace;'>
                     {i}&ensp;
                     {type(p).__name__}(id=<span style='color: black;'>{
-                        p.properties['id']
+                        p.properties["id"]
                     }</span>, provider={p.provider})
                 </summary>
                 {p._repr_html_()}
@@ -214,13 +212,12 @@ class SearchResult(UserList):
         return super().extend(other)
 
 
-class RawSearchResult(UserList):
+class RawSearchResult(UserList[dict[str, Any]]):
     """An object representing a collection of raw/unparsed search results obtained from a provider.
 
     :param results: A list of raw/unparsed search results
     """
 
-    data: list[Any]
     query_params: dict[str, Any]
     product_type_def_params: dict[str, Any]
 

eodag/config.py

@@ -615,6 +615,12 @@ class PluginConfig(yaml.YAMLObject):
     #: :class:`~eodag.plugins.authentication.token.TokenAuth`
     #: type of the token
     token_type: str
+    #: :class:`~eodag.plugins.authentication.token.TokenAuth`
+    #: key to get the expiration time of the token
+    token_expiration_key: str
+    #: :class:`~eodag.plugins.authentication.token.TokenAuth`
+    #: HTTP method to use
+    request_method: str
     #: :class:`~eodag.plugins.authentication.token_exchange.OIDCTokenExchangeAuth`
     #: The full :class:`~eodag.plugins.authentication.openid_connect.OIDCAuthorizationCodeFlowAuth` plugin configuration
     #: used to retrieve subject token

eodag/plugins/authentication/token.py

@@ -18,6 +18,8 @@
 from __future__ import annotations
 
 import logging
+from datetime import datetime, timedelta
+from threading import Lock
 from typing import TYPE_CHECKING, Any, Optional
 from urllib.parse import parse_qs, urlencode, urlparse, urlunparse
 
@@ -72,8 +74,11 @@ class TokenAuth(Authentication):
           key to get the access token in the response to the token request
         * :attr:`~eodag.config.PluginConfig.refresh_token_key` (``str``): key to get the refresh
           token in the response to the token request
+        * :attr:`~eodag.config.PluginConfig.token_expiration_key` (``str``): key to get expiration time of
+          the token (given in s)
         * :attr:`~eodag.config.PluginConfig.ssl_verify` (``bool``): if the ssl certificates
           should be verified in the requests; default: ``True``
+        * :attr:`~eodag.config.PluginConfig.request_method` (``str``): HTTP method to use; default: ``POST``
         * :attr:`~eodag.config.PluginConfig.auth_error_code` (``int``): which error code is
           returned in case of an authentication error
         * :attr:`~eodag.config.PluginConfig.req_data` (``dict[str, Any]``): if the credentials
@@ -91,6 +96,20 @@ class TokenAuth(Authentication):
         super(TokenAuth, self).__init__(provider, config)
         self.token = ""
         self.refresh_token = ""
+        self.token_expiration = datetime.now()
+        self.auth_lock = Lock()
+
+    def __getstate__(self):
+        """Exclude attributes that can't be pickled from serialization."""
+        state = dict(self.__dict__)
+        del state["auth_lock"]
+        return state
+
+    def __setstate__(self, state):
+        """Exclude attributes that can't be pickled from deserialization."""
+        self.__dict__.update(state)
+        # Init them manually
+        self.auth_lock = Lock()
 
     def validate_config_credentials(self) -> None:
         """Validate configured credentials"""
@@ -130,55 +149,72 @@ class TokenAuth(Authentication):
 
     def authenticate(self) -> AuthBase:
         """Authenticate"""
-        self.validate_config_credentials()
 
-        s = requests.Session()
-        try:
-            # First get the token
-            response = self._token_request(session=s)
-            response.raise_for_status()
-        except requests.exceptions.Timeout as exc:
-            raise TimeOutError(exc, timeout=HTTP_REQ_TIMEOUT) from exc
-        except RequestException as e:
-            response_text = getattr(e.response, "text", "").strip()
-            # check if error is identified as auth_error in provider conf
-            auth_errors = getattr(self.config, "auth_error_code", [None])
-            if not isinstance(auth_errors, list):
-                auth_errors = [auth_errors]
-            if (
-                e.response is not None
-                and getattr(e.response, "status_code", None)
-                and e.response.status_code in auth_errors
-            ):
-                raise AuthenticationError(
-                    f"Please check your credentials for {self.provider}.",
-                    f"HTTP Error {e.response.status_code} returned.",
-                    response_text,
-                ) from e
-            # other error
-            else:
-                raise AuthenticationError(
-                    "Could no get authentication token", str(e), response_text
-                ) from e
-        else:
-            if getattr(self.config, "token_type", "text") == "json":
-                token = response.json()[self.config.token_key]
+        # Use a thread lock to avoid several threads requesting the token at the same time
+        with self.auth_lock:
+
+            self.validate_config_credentials()
+            if self.token and self.token_expiration > datetime.now():
+                logger.debug("using existing access token")
+                return RequestsTokenAuth(
+                    self.token, "header", headers=getattr(self.config, "headers", {})
+                )
+            s = requests.Session()
+            try:
+                # First get the token
+                response = self._token_request(session=s)
+                response.raise_for_status()
+            except requests.exceptions.Timeout as exc:
+                raise TimeOutError(exc, timeout=HTTP_REQ_TIMEOUT) from exc
+            except RequestException as e:
+                response_text = getattr(e.response, "text", "").strip()
+                # check if error is identified as auth_error in provider conf
+                auth_errors = getattr(self.config, "auth_error_code", [None])
+                if not isinstance(auth_errors, list):
+                    auth_errors = [auth_errors]
+                if (
+                    e.response is not None
+                    and getattr(e.response, "status_code", None)
+                    and e.response.status_code in auth_errors
+                ):
+                    raise AuthenticationError(
+                        f"Please check your credentials for {self.provider}.",
+                        f"HTTP Error {e.response.status_code} returned.",
+                        response_text,
+                    ) from e
+                # other error
+                else:
+                    raise AuthenticationError(
+                        "Could no get authentication token", str(e), response_text
+                    ) from e
             else:
-                token = response.text
-            self.token = token
-            if getattr(self.config, "refresh_token_key", None):
-                self.refresh_token = response.json()[self.config.refresh_token_key]
-            if not hasattr(self.config, "headers"):
-                raise MisconfiguredError(f"Missing headers configuration for {self}")
-            # Return auth class set with obtained token
-            return RequestsTokenAuth(
-                token, "header", headers=getattr(self.config, "headers", {})
-            )
+                if getattr(self.config, "token_type", "text") == "json":
+                    token = response.json()[self.config.token_key]
+                else:
+                    token = response.text
+                self.token = token
+                if getattr(self.config, "refresh_token_key", None):
+                    self.refresh_token = response.json()[self.config.refresh_token_key]
+                if getattr(self.config, "token_expiration_key", None):
+                    expiration_time = response.json()[self.config.token_expiration_key]
+                    self.token_expiration = datetime.now() + timedelta(
+                        seconds=expiration_time
+                    )
+
+                if not hasattr(self.config, "headers"):
+                    raise MisconfiguredError(
+                        f"Missing headers configuration for {self}"
+                    )
+                # Return auth class set with obtained token
+                return RequestsTokenAuth(
+                    token, "header", headers=getattr(self.config, "headers", {})
+                )
 
     def _token_request(
         self,
         session: requests.Session,
     ) -> requests.Response:
+
         retry_total = getattr(self.config, "retry_total", REQ_RETRY_TOTAL)
         retry_backoff_factor = getattr(
             self.config, "retry_backoff_factor", REQ_RETRY_BACKOFF_FACTOR
@@ -202,14 +238,36 @@ class TokenAuth(Authentication):
         # append headers to req if some are specified in config
         req_kwargs: dict[str, Any] = {"headers": dict(headers, **USER_AGENT)}
         ssl_verify = getattr(self.config, "ssl_verify", True)
+        method = getattr(self.config, "request_method", "POST")
+
+        def set_request_data(call_refresh: bool) -> None:
+            """Set the request data contents for POST requests"""
+            if method != "POST":
+                return
+
+            # append req_data to credentials if specified in config
+            data = dict(getattr(self.config, "req_data", {}), **self.config.credentials)
+
+            # when refreshing the token, we pass only the client_id/secret if present,
+            # not other parameters (username/password, scope, ...)
+            if call_refresh:
+                data = {
+                    k: v for k, v in data.items() if k in ["client_id", "client_secret"]
+                }
+                # the grant type is always refresh_token
+                data["grant_type"] = "refresh_token"
+                # and we add the old refresh token value to the request
+                data["refresh_token"] = self.refresh_token
+
+            req_kwargs["data"] = data
 
         if self.refresh_token:
             logger.debug("fetching access token with refresh token")
             session.mount(self.config.refresh_uri, HTTPAdapter(max_retries=retries))
+            set_request_data(call_refresh=True)
             try:
                 response = session.post(
                     self.config.refresh_uri,
-                    data={"refresh_token": self.refresh_token},
                     timeout=HTTP_REQ_TIMEOUT,
                     verify=ssl_verify,
                     **req_kwargs,
@@ -222,14 +280,8 @@ class TokenAuth(Authentication):
         logger.debug("fetching access token from %s", self.config.auth_uri)
         # append headers to req if some are specified in config
         session.mount(self.config.auth_uri, HTTPAdapter(max_retries=retries))
-        method = getattr(self.config, "request_method", "POST")
 
-        # send credentials also as data in POST requests
-        if method == "POST":
-            # append req_data to credentials if specified in config
-            req_kwargs["data"] = dict(
-                getattr(self.config, "req_data", {}), **self.config.credentials
-            )
+        set_request_data(call_refresh=False)
 
         # credentials as auth tuple if possible
         req_kwargs["auth"] = (

eodag/plugins/download/http.py

@@ -216,7 +216,7 @@ class HTTPDownload(Download):
                     product.properties["storageStatus"] = STAGING_STATUS
                 except RequestException as e:
                     self._check_auth_exception(e)
-                    msg = f'{product.properties["title"]} could not be ordered'
+                    msg = f"{product.properties['title']} could not be ordered"
                     if e.response is not None and e.response.status_code == 400:
                         raise ValidationError.from_error(e, msg) from e
                     else:
@@ -255,6 +255,16 @@ class HTTPDownload(Download):
         product.properties.update(
             {k: v for k, v in properties_update.items() if v != NOT_AVAILABLE}
         )
+        # the job id becomes the product id for EcmwfSearch products
+        if "ORDERABLE" in product.properties.get("id", ""):
+            product.properties["id"] = product.properties.get(
+                "orderId", product.properties["id"]
+            )
+            product.properties["title"] = (
+                (product.product_type or product.provider).upper()
+                + "_"
+                + product.properties["id"]
+            )
         if "downloadLink" in product.properties:
             product.remote_location = product.location = product.properties[
                 "downloadLink"
@@ -390,7 +400,10 @@ class HTTPDownload(Download):
                     # success and no need to get status response content
                     skip_parsing_status_response = True
             except RequestException as e:
-                msg = f'{product.properties["title"]} order status could not be checked'
+                msg = (
+                    f"{product.properties.get('title') or product.properties.get('id') or product} "
+                    "order status could not be checked"
+                )
                 if e.response is not None and e.response.status_code == 400:
                     raise ValidationError.from_error(e, msg) from e
                 else:
@@ -426,9 +439,14 @@ class HTTPDownload(Download):
                     f"{product.properties['title']} order status: {status_percent}"
                 )
 
-            status_message = status_dict.get("message")
+            product.properties.update(
+                {k: v for k, v in status_dict.items() if v != NOT_AVAILABLE}
+            )
+
             product.properties["orderStatus"] = status_dict.get("status")
 
+            status_message = status_dict.get("message")
+
             # handle status error
             errors: dict[str, Any] = status_config.get("error", {})
             if errors and errors.items() <= status_dict.items():
@@ -436,13 +454,14 @@ class HTTPDownload(Download):
                     f"Provider {product.provider} returned: {status_dict.get('error_message', status_message)}"
                 )
 
+        product.properties["storageStatus"] = STAGING_STATUS
+
         success_status: dict[str, Any] = status_config.get("success", {}).get("status")
         # if not success
         if (success_status and success_status != status_dict.get("status")) or (
             success_code and success_code != response.status_code
         ):
-            error = NotAvailableError(status_message)
-            raise error
+            return None
 
         product.properties["storageStatus"] = ONLINE_STATUS
 
@@ -461,7 +480,11 @@ class HTTPDownload(Download):
                     product.properties["title"],
                     e,
                 )
-                return None
+                msg = f"{product.properties['title']} order status could not be checked"
+                if e.response is not None and e.response.status_code == 400:
+                    raise ValidationError.from_error(e, msg) from e
+                else:
+                    raise DownloadError.from_error(e, msg) from e
 
         result_type = config_on_success.get("result_type", "json")
         result_entry = config_on_success.get("results_entry")
@@ -626,6 +649,8 @@ class HTTPDownload(Download):
             if fs_path is not None:
                 ext = Path(product.filename).suffix
                 path = Path(fs_path).with_suffix(ext)
+                if "ORDERABLE" in path.stem and product.properties.get("title"):
+                    path = path.with_stem(sanitize(product.properties["title"]))
 
                 with open(path, "wb") as fhandle:
                     for chunk in chunk_iterator:
@@ -961,17 +986,21 @@ class HTTPDownload(Download):
             auth = None
 
         s = requests.Session()
-        self.stream = s.request(
-            req_method,
-            req_url,
-            stream=True,
-            auth=auth,
-            params=params,
-            headers=USER_AGENT,
-            timeout=DEFAULT_STREAM_REQUESTS_TIMEOUT,
-            verify=ssl_verify,
-            **req_kwargs,
-        )
+        try:
+            self.stream = s.request(
+                req_method,
+                req_url,
+                stream=True,
+                auth=auth,
+                params=params,
+                headers=USER_AGENT,
+                timeout=DEFAULT_STREAM_REQUESTS_TIMEOUT,
+                verify=ssl_verify,
+                **req_kwargs,
+            )
+        except requests.exceptions.MissingSchema:
+            # location is not a valid url -> product is not available yet
+            raise NotAvailableError("Product is not available yet")
         try:
             self.stream.raise_for_status()
         except requests.exceptions.Timeout as exc:

eodag/plugins/search/base.py

@@ -337,7 +337,9 @@ class Search(PluginTopic):
         try:
             filters["productType"] = product_type
             queryables = self.discover_queryables(**{**default_values, **filters}) or {}
-        except NotImplementedError:
+        except NotImplementedError as e:
+            if str(e):
+                logger.debug(str(e))
             queryables = self.queryables_from_metadata_mapping(product_type, alias)
 
         return QueryablesDict(**queryables)
@@ -381,12 +383,12 @@ class Search(PluginTopic):
             for pt in available_product_types:
                 self.config.product_type_config = product_type_configs[pt]
                 pt_queryables = self._get_product_type_queryables(pt, None, filters)
-                # only use key and type because values and defaults will vary between product types
-                pt_queryables_neutral = {
-                    k: Annotated[v.__args__[0], Field(default=None)]
-                    for k, v in pt_queryables.items()
-                }
-                all_queryables.update(pt_queryables_neutral)
+                all_queryables.update(pt_queryables)
+            # reset defaults because they may vary between product types
+            for k, v in all_queryables.items():
+                v.__metadata__[0].default = getattr(
+                    Queryables.model_fields.get(k, Field(None)), "default", None
+                )
             return QueryablesDict(
                 additional_properties=True,
                 additional_information=additional_info,