eodag 3.0.0b3__py3-none-any.whl → 3.1.0b1__py3-none-any.whl

eodag/plugins/apis/base.py

@@ -24,9 +24,10 @@ from eodag.plugins.search.base import Search
 class Api(Search, Download):
     """Plugins API Base plugin
 
-    An Api plugin inherit the methods from Search and Download plugins.
+    An Api plugin inherits the methods from Search and Download plugins.
 
     There are three methods that it must implement:
+
     - ``query``: search for products
     - ``download``: download a single :class:`~eodag.api.product._product.EOProduct`
     - ``download_all``: download multiple products from a :class:`~eodag.api.search_result.SearchResult`
@@ -35,14 +36,14 @@ class Api(Search, Download):
 
     - download data in the ``output_dir`` folder defined in the plugin's
       configuration or passed through kwargs
-    - extract products from their archive (if relevant) if ``extract`` is set to True
-      (True by default)
+    - extract products from their archive (if relevant) if ``extract`` is set to ``True``
+      (``True`` by default)
     - save a product in an archive/directory (in ``output_dir``) whose name must be
       the product's ``title`` property
     - update the product's ``location`` attribute once its data is downloaded (and
       eventually after it's extracted) to the product's location given as a file URI
       (e.g. 'file:///tmp/product_folder' on Linux or
-      'file:///C:/Users/username/AppData/LOcal/Temp' on Windows)
+      'file:///C:/Users/username/AppData/Local/Temp' on Windows)
     - save a *record* file in the directory ``output_dir/.downloaded`` whose name
       is built on the MD5 hash of the product's ``product_type`` and ``properties['id']``
       attributes (``hashlib.md5((product.product_type+"-"+product.properties['id']).encode("utf-8")).hexdigest()``)
@@ -52,4 +53,9 @@ class Api(Search, Download):
     - not try to download a product if its *record* file exists as long as the expected
       product's file/directory. If the *record* file only is found, it must be deleted
       (it certainly indicates that the download didn't complete)
+
+    :param provider: An EODAG provider name
+    :type provider: str
+    :param config: An EODAG plugin configuration
+    :type config: Dict[str, Any]
     """

eodag/plugins/apis/ecmwf.py

@@ -20,16 +20,21 @@ from __future__ import annotations
 import logging
 import os
 from datetime import datetime, timezone
-from typing import TYPE_CHECKING
+from typing import TYPE_CHECKING, Annotated
 
 import geojson
 from ecmwfapi import ECMWFDataServer, ECMWFService
 from ecmwfapi.api import APIException, Connection, get_apikey_values
+from pydantic.fields import FieldInfo
 
 from eodag.plugins.apis.base import Api
 from eodag.plugins.search import PreparedSearch
 from eodag.plugins.search.base import Search
-from eodag.plugins.search.build_search_result import BuildPostSearchResult
+from eodag.plugins.search.build_search_result import (
+    ECMWF_KEYWORDS,
+    ECMWFSearch,
+    keywords_to_mdt,
+)
 from eodag.utils import (
     DEFAULT_DOWNLOAD_TIMEOUT,
     DEFAULT_DOWNLOAD_WAIT,
@@ -58,7 +63,7 @@ logger = logging.getLogger("eodag.apis.ecmwf")
 ECMWF_MARS_KNOWN_FORMATS = {"grib": "grib", "netcdf": "nc"}
 
 
-class EcmwfApi(Api, BuildPostSearchResult):
+class EcmwfApi(Api, ECMWFSearch):
     """A plugin that enables to build download-request and download data on ECMWF MARS.
 
     Builds a single ready-to-download :class:`~eodag.api.product._product.EOProduct`
@@ -68,20 +73,36 @@ class EcmwfApi(Api, BuildPostSearchResult):
     is in query), or on MARS Operational Archive (if ``dataset`` parameter is not in
     query).
 
-    This class inherits from :class:`~eodag.plugins.apis.base.Api` for compatibility,
-    :class:`~eodag.plugins.download.base.Download` for download methods, and
-    :class:`~eodag.plugins.search.qssearch.QueryStringSearch` for metadata-mapping and
-    query build methods.
+    This class inherits from :class:`~eodag.plugins.apis.base.Api` for compatibility and
+    :class:`~eodag.plugins.search.build_search_result.ECMWFSearch` for the creation
+    of the search result.
+
+    :param provider: provider name
+    :param config: Api plugin configuration:
+
+        * :attr:`~eodag.config.PluginConfig.type` (``str``) (**mandatory**): EcmwfApi
+        * :attr:`~eodag.config.PluginConfig.auth_endpoint` (``str``) (**mandatory**): url of
+          the authentication endpoint of the ecmwf api
+        * :attr:`~eodag.config.PluginConfig.metadata_mapping` (``Dict[str, Union[str, list]]``): how
+          parameters should be mapped between the provider and eodag; If a string is given, this is
+          the mapping parameter returned by provider -> eodag parameter. If a list with 2 elements
+          is given, the first one is the mapping eodag parameter -> provider query parameters
+          and the second one the mapping provider result parameter -> eodag parameter
     """
 
     def __init__(self, provider: str, config: PluginConfig) -> None:
         # init self.config.metadata_mapping using Search Base plugin
+        config.metadata_mapping = {
+            **keywords_to_mdt(ECMWF_KEYWORDS, "ecmwf"),
+            **config.metadata_mapping,
+        }
         Search.__init__(self, provider, config)
 
         # needed by QueryStringSearch.build_query_string / format_free_text_search
         self.config.__dict__.setdefault("free_text_search_operations", {})
         # needed for compatibility
         self.config.__dict__.setdefault("pagination", {"next_page_query_obj": "{{}}"})
+        self.config.__dict__.setdefault("api_endpoint", "")
 
     def do_search(self, *args: Any, **kwargs: Any) -> List[Dict[str, Any]]:
         """Should perform the actual search request."""
@@ -98,9 +119,9 @@ class EcmwfApi(Api, BuildPostSearchResult):
         # productType
         if not kwargs.get("productType"):
             kwargs["productType"] = "%s_%s_%s" % (
-                kwargs.get("dataset", "mars"),
-                kwargs.get("type", ""),
-                kwargs.get("levtype", ""),
+                kwargs.get("ecmwf:dataset", "mars"),
+                kwargs.get("ecmwf:type", ""),
+                kwargs.get("ecmwf:levtype", ""),
             )
         # start date
         if "startTimeFromAscendingNode" not in kwargs:
@@ -122,7 +143,7 @@ class EcmwfApi(Api, BuildPostSearchResult):
         if "geometry" in kwargs:
             kwargs["geometry"] = get_geometry_from_various(geometry=kwargs["geometry"])
 
-        return BuildPostSearchResult.query(self, prep, **kwargs)
+        return ECMWFSearch.query(self, prep, **kwargs)
 
     def authenticate(self) -> Dict[str, Optional[str]]:
         """Check credentials and returns information needed for auth
@@ -133,7 +154,7 @@ class EcmwfApi(Api, BuildPostSearchResult):
         # Get credentials from eodag or using ecmwf conf
         email = getattr(self.config, "credentials", {}).get("username", None)
         key = getattr(self.config, "credentials", {}).get("password", None)
-        url = getattr(self.config, "api_endpoint", None)
+        url = getattr(self.config, "auth_endpoint", None)
         if not all([email, key, url]):
             key, url, email = get_apikey_values()
 
@@ -157,8 +178,8 @@ class EcmwfApi(Api, BuildPostSearchResult):
         product: EOProduct,
         auth: Optional[Union[AuthBase, Dict[str, str]]] = None,
         progress_callback: Optional[ProgressCallback] = None,
-        wait: int = DEFAULT_DOWNLOAD_WAIT,
-        timeout: int = DEFAULT_DOWNLOAD_TIMEOUT,
+        wait: float = DEFAULT_DOWNLOAD_WAIT,
+        timeout: float = DEFAULT_DOWNLOAD_TIMEOUT,
         **kwargs: Unpack[DownloadConf],
     ) -> Optional[str]:
         """Download data from ECMWF MARS"""
@@ -247,8 +268,8 @@ class EcmwfApi(Api, BuildPostSearchResult):
         auth: Optional[Union[AuthBase, Dict[str, str]]] = None,
         downloaded_callback: Optional[DownloadedCallback] = None,
         progress_callback: Optional[ProgressCallback] = None,
-        wait: int = DEFAULT_DOWNLOAD_WAIT,
-        timeout: int = DEFAULT_DOWNLOAD_TIMEOUT,
+        wait: float = DEFAULT_DOWNLOAD_WAIT,
+        timeout: float = DEFAULT_DOWNLOAD_TIMEOUT,
         **kwargs: Unpack[DownloadConf],
     ) -> List[str]:
         """
@@ -267,3 +288,15 @@ class EcmwfApi(Api, BuildPostSearchResult):
     def clear(self) -> None:
         """Clear search context"""
         pass
+
+    def discover_queryables(
+        self, **kwargs: Any
+    ) -> Optional[Dict[str, Annotated[Any, FieldInfo]]]:
+        """Fetch queryables list from provider using metadata mapping
+
+        :param kwargs: additional filters for queryables (`productType` and other search
+                       arguments)
+        :returns: fetched queryable parameters dict
+        """
+        product_type = kwargs.get("productType", None)
+        return self.queryables_from_metadata_mapping(product_type)

eodag/plugins/apis/usgs.py

@@ -68,7 +68,30 @@ logger = logging.getLogger("eodag.apis.usgs")
 
 
 class UsgsApi(Api):
-    """A plugin that enables to query and download data on the USGS catalogues"""
+    """A plugin that enables to query and download data on the USGS catalogues
+
+    :param provider: provider name
+    :param config: Api plugin configuration:
+
+        * :attr:`~eodag.config.PluginConfig.type` (``str``) (**mandatory**): UsgsApi
+        * :attr:`~eodag.config.PluginConfig.pagination` (:class:`~eodag.config.PluginConfig.Pagination`)
+          (**mandatory**): object containing parameters for pagination; should contain the attribute
+          :attr:`~eodag.config.PluginConfig.Pagination.total_items_nb_key_path`
+          which is indicating the key for the number of total items in the provider result
+        * :attr:`~eodag.config.PluginConfig.ssl_verify` (``bool``): if the ssl certificates
+          should be verified in the download request; default: ``True``
+        * :attr:`~eodag.config.PluginConfig.need_auth` (``bool``): if authentication is required
+          for search; default: ``False``
+        * :attr:`~eodag.config.PluginConfig.extract` (``bool``): if the content of the downloaded
+          file should be extracted; default: ``True``
+        * :attr:`~eodag.config.PluginConfig.order_enabled` (``bool``): if the product has to
+          be ordered to download it; default: ``False``
+        * :attr:`~eodag.config.PluginConfig.metadata_mapping` (``Dict[str, Union[str, list]]``): how
+          parameters should be mapped between the provider and eodag; If a string is given, this is
+          the mapping parameter returned by provider -> eodag parameter. If a list with 2 elements
+          is given, the first one is the mapping eodag parameter -> provider query parameters
+          and the second one the mapping provider result parameter -> eodag parameter
+    """
 
     def __init__(self, provider: str, config: PluginConfig) -> None:
         super(UsgsApi, self).__init__(provider, config)
@@ -256,7 +279,7 @@ class UsgsApi(Api):
                 f"Product type {usgs_dataset} may not exist on USGS EE catalog"
             )
             api.logout()
-            raise RequestError(e)
+            raise RequestError.from_error(e) from e
 
         api.logout()
 
@@ -274,8 +297,8 @@ class UsgsApi(Api):
         product: EOProduct,
         auth: Optional[Union[AuthBase, Dict[str, str]]] = None,
         progress_callback: Optional[ProgressCallback] = None,
-        wait: int = DEFAULT_DOWNLOAD_WAIT,
-        timeout: int = DEFAULT_DOWNLOAD_TIMEOUT,
+        wait: float = DEFAULT_DOWNLOAD_WAIT,
+        timeout: float = DEFAULT_DOWNLOAD_TIMEOUT,
         **kwargs: Unpack[DownloadConf],
     ) -> Optional[str]:
         """Download data from USGS catalogues"""
@@ -352,7 +375,7 @@ class UsgsApi(Api):
         logger.debug(f"Downloading {req_url}")
         ssl_verify = getattr(self.config, "ssl_verify", True)
 
-        @self._download_retry(product, wait, timeout)
+        @self._order_download_retry(product, wait, timeout)
         def download_request(
             product: EOProduct,
             fs_path: str,
@@ -444,8 +467,8 @@ class UsgsApi(Api):
         auth: Optional[Union[AuthBase, Dict[str, str]]] = None,
         downloaded_callback: Optional[DownloadedCallback] = None,
         progress_callback: Optional[ProgressCallback] = None,
-        wait: int = DEFAULT_DOWNLOAD_WAIT,
-        timeout: int = DEFAULT_DOWNLOAD_TIMEOUT,
+        wait: float = DEFAULT_DOWNLOAD_WAIT,
+        timeout: float = DEFAULT_DOWNLOAD_TIMEOUT,
         **kwargs: Unpack[DownloadConf],
     ) -> List[str]:
         """

eodag/plugins/authentication/aws_auth.py

@@ -30,14 +30,23 @@ if TYPE_CHECKING:
 class AwsAuth(Authentication):
     """AWS authentication plugin
 
-    Authentication will use the first valid method within the following ones:
+    Authentication will use the first valid method within the following ones depending on which
+    parameters are available in the configuration:
 
-    - auth anonymously using no-sign-request
-    - auth using ``aws_profile``
-    - auth using ``aws_access_key_id`` and ``aws_secret_access_key``
+    * auth anonymously using no-sign-request
+    * auth using ``aws_profile``
+    * auth using ``aws_access_key_id`` and ``aws_secret_access_key``
       (optionally ``aws_session_token``)
-    - auth using current environment (AWS environment variables and/or ``~/aws/*``),
+    * auth using current environment (AWS environment variables and/or ``~/aws/*``),
       will be skipped if AWS credentials are filled in eodag conf
+
+    :param provider: provider name
+    :param config: Authentication plugin configuration:
+
+        * :attr:`~eodag.config.PluginConfig.type` (``str``) (**mandatory**): AwsAuth
+        * :attr:`~eodag.config.PluginConfig.auth_error_code` (``int``) (mandatory for ``creodias_s3``):
+          which error code is returned in case of an authentication error
+
     """
 
     s3_client: S3Client

eodag/plugins/authentication/base.py

@@ -27,7 +27,16 @@ if TYPE_CHECKING:
 
 
 class Authentication(PluginTopic):
-    """Plugins authentication Base plugin"""
+    """Plugins authentication Base plugin
+
+    :param provider: provider name
+    :param config: Authentication plugin configuration:
+
+        * :attr:`~eodag.config.PluginConfig.matching_url` (``str``): URL pattern to match with search plugin endpoint or
+          download link
+        * :attr:`~eodag.config.PluginConfig.matching_conf` (``Dict[str, Any]``): Part of the search or download plugin
+          configuration that needs authentication and helps identifying it
+    """
 
     def authenticate(self) -> Union[AuthBase, Dict[str, str]]:
         """Authenticate"""

eodag/plugins/authentication/generic.py

@@ -29,7 +29,18 @@ if TYPE_CHECKING:
 
 
 class GenericAuth(Authentication):
-    """GenericAuth authentication plugin"""
+    """GenericAuth authentication plugin (authentication using ``username`` and ``password``)
+
+    The mandatory parameters that have to be added in the eodag config are username and password.
+
+    :param provider: provider name
+    :param config: Authentication plugin configuration:
+
+        * :attr:`~eodag.config.PluginConfig.type` (``str``) (**mandatory**): GenericAuth
+        * :attr:`~eodag.config.PluginConfig.method` (``str``): specifies if digest authentication
+          (``digest``) or basic authentication (``basic``) should be used; default: ``basic``
+
+    """
 
     def authenticate(self) -> AuthBase:
         """Authenticate"""
@@ -48,6 +59,6 @@ class GenericAuth(Authentication):
             )
         else:
             raise MisconfiguredError(
-                f"Cannot authenticate with {self.provider}:",
-                f"method {method} is not supported. Must be one of digest or basic",
+                f"Cannot authenticate with {self.provider}",
+                f"Method {method} is not supported; it must be one of 'digest' or 'basic'.",
             )

eodag/plugins/authentication/header.py

@@ -34,7 +34,14 @@ class HTTPHeaderAuth(Authentication):
     This plugin enables implementation of custom HTTP authentication scheme (other than Basic, Digest, Token
     negotiation et al.) using HTTP headers.
 
-    The plugin is configured as follows in the providers config file::
+    :param provider: provider name
+    :param config: Authentication plugin configuration:
+
+        * :attr:`~eodag.config.PluginConfig.type` (``str``) (**mandatory**): HTTPHeaderAuth
+        * :attr:`~eodag.config.PluginConfig.headers` (``Dict[str, str]``): dictionary containing
+          all keys/value pairs that should be added to the headers
+
+    Below an example for the configuration in the providers config file is shown::
 
         provider:
             ...
@@ -47,9 +54,9 @@ class HTTPHeaderAuth(Authentication):
                 ...
             ...
 
-    As you can see in the sample above, the maintainer of `provider` define the headers that will be used in the
-    authentication process as-is, by giving their names (e.g. `Authorization`) and their value (e.g
-    `"Something {userinput}"`) as regular Python string templates that enable passing in the user input necessary to
+    As you can see in the sample above, the maintainer of ``provider`` define the headers that will be used in the
+    authentication process as-is, by giving their names (e.g. ``Authorization``) and their value (e.g
+    ``"Something {userinput}"``) as regular Python string templates that enable passing in the user input necessary to
     compute its identity. The user input awaited in the header value string must be present in the user config file.
     In the sample above, the plugin await for user credentials to be specified as::
 
@@ -72,6 +79,7 @@ class HTTPHeaderAuth(Authentication):
                     X-Another-Special-Header: "YYY"
                 ...
             ...
+
     """
 
     def authenticate(self) -> HeaderAuth:

eodag/plugins/authentication/keycloak.py

@@ -41,19 +41,38 @@ logger = logging.getLogger("eodag.auth.keycloak")
 class KeycloakOIDCPasswordAuth(OIDCRefreshTokenBase):
     """Authentication plugin using Keycloak and OpenId Connect.
 
-    This plugin request a token and use it through a query-string or a header.
+    This plugin requests a token which is added to a query-string or a header for authentication.
+
+    :param provider: provider name
+    :param config: Authentication plugin configuration:
+
+        * :attr:`~eodag.config.PluginConfig.type` (``str``) (**mandatory**): KeycloakOIDCPasswordAuth
+        * :attr:`~eodag.config.PluginConfig.oidc_config_url` (``str``) (**mandatory**):
+          The url to get the OIDC Provider's endpoints
+        * :attr:`~eodag.config.PluginConfig.client_id` (``str``) (**mandatory**): keycloak client id
+        * :attr:`~eodag.config.PluginConfig.client_secret` (``str``) (**mandatory**): keycloak
+          client secret, set to null if no secret is used
+        * :attr:`~eodag.config.PluginConfig.token_provision` (``str``) (**mandatory**): if the
+          token should be added to the query string (``qs``) or to the header (``header``)
+        * :attr:`~eodag.config.PluginConfig.token_qs_key` (``str``): (**mandatory if token_provision=qs**)
+          key of the param added to the query string
+        * :attr:`~eodag.config.PluginConfig.allowed_audiences` (``List[str]``) (**mandatory**):
+          The allowed audiences that have to be present in the user token.
+        * :attr:`~eodag.config.PluginConfig.auth_error_code` (``int``): which error code is
+          returned in case of an authentication error
+        * :attr:`~eodag.config.PluginConfig.ssl_verify` (``bool``): if the ssl certificates
+          should be verified in the token request; default: ``True``
 
     Using :class:`~eodag.plugins.download.http.HTTPDownload` a download link
-    `http://example.com?foo=bar` will become
-    `http://example.com?foo=bar&my-token=obtained-token` if associated to the following
+    ``http://example.com?foo=bar`` will become
+    ``http://example.com?foo=bar&my-token=obtained-token`` if associated to the following
     configuration::
 
         provider:
             ...
             auth:
                 plugin: KeycloakOIDCPasswordAuth
-                auth_base_uri: 'https://somewhere/auth'
-                realm: 'the-realm'
+                oidc_config_url: 'https://somewhere/auth/realms/realm/.well-known/openid-configuration'
                 client_id: 'SOME_ID'
                 client_secret: '01234-56789'
                 token_provision: qs
@@ -62,15 +81,14 @@ class KeycloakOIDCPasswordAuth(OIDCRefreshTokenBase):
             ...
 
     If configured to send the token through the header, the download request header will
-    be updated with `Authorization: "Bearer obtained-token"` if associated to the
+    be updated with ``Authorization: "Bearer obtained-token"`` if associated to the
     following configuration::
 
         provider:
             ...
             auth:
                 plugin: KeycloakOIDCPasswordAuth
-                auth_base_uri: 'https://somewhere/auth'
-                realm: 'the-realm'
+                oidc_config_url: 'https://somewhere/auth/realms/realm/.well-known/openid-configuration'
                 client_id: 'SOME_ID'
                 client_secret: '01234-56789'
                 token_provision: header
@@ -79,8 +97,12 @@ class KeycloakOIDCPasswordAuth(OIDCRefreshTokenBase):
     """
 
     GRANT_TYPE = "password"
-    TOKEN_URL_TEMPLATE = "{auth_base_uri}/realms/{realm}/protocol/openid-connect/token"
-    REQUIRED_PARAMS = ["auth_base_uri", "client_id", "client_secret", "token_provision"]
+    REQUIRED_PARAMS = [
+        "oidc_config_url",
+        "client_id",
+        "client_secret",
+        "token_provision",
+    ]
 
     def __init__(self, provider: str, config: PluginConfig) -> None:
         super(KeycloakOIDCPasswordAuth, self).__init__(provider, config)
@@ -101,10 +123,9 @@ class KeycloakOIDCPasswordAuth(OIDCRefreshTokenBase):
         Makes authentication request
         """
         self.validate_config_credentials()
-        access_token = self._get_access_token()
-        self.token_info["access_token"] = access_token
+        self._get_access_token()
         return CodeAuthorizedAuth(
-            self.token_info["access_token"],
+            self.access_token,
             self.config.token_provision,
             key=getattr(self.config, "token_qs_key", None),
         )
@@ -117,15 +138,14 @@ class KeycloakOIDCPasswordAuth(OIDCRefreshTokenBase):
             "grant_type": self.GRANT_TYPE,
         }
         credentials = {k: v for k, v in self.config.credentials.items()}
+        ssl_verify = getattr(self.config, "ssl_verify", True)
         try:
             response = self.session.post(
-                self.TOKEN_URL_TEMPLATE.format(
-                    auth_base_uri=self.config.auth_base_uri.rstrip("/"),
-                    realm=self.config.realm,
-                ),
+                self.token_endpoint,
                 data=dict(req_data, **credentials),
                 headers=USER_AGENT,
                 timeout=HTTP_REQ_TIMEOUT,
+                verify=ssl_verify,
             )
             response.raise_for_status()
         except requests.exceptions.Timeout as exc:
@@ -140,17 +160,16 @@ class KeycloakOIDCPasswordAuth(OIDCRefreshTokenBase):
             "client_id": self.config.client_id,
             "client_secret": self.config.client_secret,
             "grant_type": "refresh_token",
-            "refresh_token": self.token_info["refresh_token"],
+            "refresh_token": self.refresh_token,
         }
+        ssl_verify = getattr(self.config, "ssl_verify", True)
         try:
             response = self.session.post(
-                self.TOKEN_URL_TEMPLATE.format(
-                    auth_base_uri=self.config.auth_base_uri.rstrip("/"),
-                    realm=self.config.realm,
-                ),
+                self.token_endpoint,
                 data=req_data,
                 headers=USER_AGENT,
                 timeout=HTTP_REQ_TIMEOUT,
+                verify=ssl_verify,
             )
             response.raise_for_status()
         except requests.RequestException as e:

eodag/plugins/authentication/oauth.py

@@ -26,7 +26,17 @@ if TYPE_CHECKING:
 
 
 class OAuth(Authentication):
-    """OAuth authentication plugin"""
+    """OAuth authentication plugin
+
+    The mandatory parameters that have to be added in the eodag config are ``aws_access_key_id``
+    and ``aws_secret_access_key``.
+
+    :param provider: provider name
+    :param config: Authentication plugin configuration:
+
+        * :attr:`~eodag.config.PluginConfig.type` (``str``) (**mandatory**): OAuth
+
+    """
 
     def __init__(self, provider: str, config: PluginConfig) -> None:
         super(OAuth, self).__init__(provider, config)