eodag 3.0.0b2__py3-none-any.whl → 3.0.1__py3-none-any.whl

eodag/plugins/apis/base.py

@@ -24,26 +24,27 @@ from eodag.plugins.search.base import Search
 class Api(Search, Download):
     """Plugins API Base plugin
 
-    An Api plugin inherit the methods from Search and Download plugins.
+    An Api plugin inherits the methods from Search and Download plugins.
 
     There are three methods that it must implement:
+
     - ``query``: search for products
     - ``download``: download a single :class:`~eodag.api.product._product.EOProduct`
     - ``download_all``: download multiple products from a :class:`~eodag.api.search_result.SearchResult`
 
     The download methods must:
 
-    - download data in the ``outputs_prefix`` folder defined in the plugin's
+    - download data in the ``output_dir`` folder defined in the plugin's
       configuration or passed through kwargs
-    - extract products from their archive (if relevant) if ``extract`` is set to True
-      (True by default)
-    - save a product in an archive/directory (in ``outputs_prefix``) whose name must be
+    - extract products from their archive (if relevant) if ``extract`` is set to ``True``
+      (``True`` by default)
+    - save a product in an archive/directory (in ``output_dir``) whose name must be
       the product's ``title`` property
     - update the product's ``location`` attribute once its data is downloaded (and
       eventually after it's extracted) to the product's location given as a file URI
       (e.g. 'file:///tmp/product_folder' on Linux or
-      'file:///C:/Users/username/AppData/LOcal/Temp' on Windows)
-    - save a *record* file in the directory ``outputs_prefix/.downloaded`` whose name
+      'file:///C:/Users/username/AppData/Local/Temp' on Windows)
+    - save a *record* file in the directory ``output_dir/.downloaded`` whose name
       is built on the MD5 hash of the product's ``product_type`` and ``properties['id']``
       attributes (``hashlib.md5((product.product_type+"-"+product.properties['id']).encode("utf-8")).hexdigest()``)
       and whose content is the product's ``remote_location`` attribute itself.
@@ -52,4 +53,9 @@ class Api(Search, Download):
     - not try to download a product if its *record* file exists as long as the expected
       product's file/directory. If the *record* file only is found, it must be deleted
       (it certainly indicates that the download didn't complete)
+
+    :param provider: An EODAG provider name
+    :type provider: str
+    :param config: An EODAG plugin configuration
+    :type config: Dict[str, Any]
     """

eodag/plugins/apis/ecmwf.py

@@ -68,10 +68,20 @@ class EcmwfApi(Api, BuildPostSearchResult):
     is in query), or on MARS Operational Archive (if ``dataset`` parameter is not in
     query).
 
-    This class inherits from :class:`~eodag.plugins.apis.base.Api` for compatibility,
-    :class:`~eodag.plugins.download.base.Download` for download methods, and
-    :class:`~eodag.plugins.search.qssearch.QueryStringSearch` for metadata-mapping and
-    query build methods.
+    This class inherits from :class:`~eodag.plugins.apis.base.Api` for compatibility and
+    :class:`~eodag.plugins.search.build_search_result.BuildPostSearchResult` for the creation
+    of the search result.
+
+    :param provider: provider name
+    :param config: Api plugin configuration:
+
+        * :attr:`~eodag.config.PluginConfig.type` (``str``) (**mandatory**): EcmwfApi
+        * :attr:`~eodag.config.PluginConfig.api_endpoint` (``str``) (**mandatory**): url of the ecmwf api
+        * :attr:`~eodag.config.PluginConfig.metadata_mapping` (``Dict[str, Union[str, list]]``): how
+          parameters should be mapped between the provider and eodag; If a string is given, this is
+          the mapping parameter returned by provider -> eodag parameter. If a list with 2 elements
+          is given, the first one is the mapping eodag parameter -> provider query parameters
+          and the second one the mapping provider result parameter -> eodag parameter
     """
 
     def __init__(self, provider: str, config: PluginConfig) -> None:
@@ -128,7 +138,6 @@ class EcmwfApi(Api, BuildPostSearchResult):
         """Check credentials and returns information needed for auth
 
         :returns: {key, url, email} dictionary
-        :rtype: dict
         :raises: :class:`~eodag.utils.exceptions.AuthenticationError`
         """
         # Get credentials from eodag or using ecmwf conf
@@ -165,8 +174,8 @@ class EcmwfApi(Api, BuildPostSearchResult):
         """Download data from ECMWF MARS"""
         product_format = product.properties.get("format", "grib")
         product_extension = ECMWF_MARS_KNOWN_FORMATS.get(product_format, product_format)
-        kwargs["outputs_extension"] = kwargs.get(
-            "outputs_extension", f".{product_extension}"
+        kwargs["output_extension"] = kwargs.get(
+            "output_extension", f".{product_extension}"
         )
 
         # Prepare download

eodag/plugins/apis/usgs.py

@@ -68,7 +68,30 @@ logger = logging.getLogger("eodag.apis.usgs")
 
 
 class UsgsApi(Api):
-    """A plugin that enables to query and download data on the USGS catalogues"""
+    """A plugin that enables to query and download data on the USGS catalogues
+
+    :param provider: provider name
+    :param config: Api plugin configuration:
+
+        * :attr:`~eodag.config.PluginConfig.type` (``str``) (**mandatory**): UsgsApi
+        * :attr:`~eodag.config.PluginConfig.pagination` (:class:`~eodag.config.PluginConfig.Pagination`)
+          (**mandatory**): object containing parameters for pagination; should contain the attribute
+          :attr:`~eodag.config.PluginConfig.Pagination.total_items_nb_key_path`
+          which is indicating the key for the number of total items in the provider result
+        * :attr:`~eodag.config.PluginConfig.ssl_verify` (``bool``): if the ssl certificates
+          should be verified in the download request; default: ``True``
+        * :attr:`~eodag.config.PluginConfig.need_auth` (``bool``): if authentication is required
+          for search; default: ``False``
+        * :attr:`~eodag.config.PluginConfig.extract` (``bool``): if the content of the downloaded
+          file should be extracted; default: ``True``
+        * :attr:`~eodag.config.PluginConfig.order_enabled` (``bool``): if the product has to
+          be ordered to download it; default: ``False``
+        * :attr:`~eodag.config.PluginConfig.metadata_mapping` (``Dict[str, Union[str, list]]``): how
+          parameters should be mapped between the provider and eodag; If a string is given, this is
+          the mapping parameter returned by provider -> eodag parameter. If a list with 2 elements
+          is given, the first one is the mapping eodag parameter -> provider query parameters
+          and the second one the mapping provider result parameter -> eodag parameter
+    """
 
     def __init__(self, provider: str, config: PluginConfig) -> None:
         super(UsgsApi, self).__init__(provider, config)
@@ -128,13 +151,14 @@ class UsgsApi(Api):
             raise NoMatchingProductType(
                 "Cannot search on USGS without productType specified"
             )
-        if kwargs.get("sortBy"):
+        if kwargs.get("sort_by"):
             raise ValidationError("USGS does not support sorting feature")
 
         self.authenticate()
 
         product_type_def_params = self.config.products.get(  # type: ignore
-            product_type, self.config.products[GENERIC_PRODUCT_TYPE]  # type: ignore
+            product_type,
+            self.config.products[GENERIC_PRODUCT_TYPE],  # type: ignore
         )
         usgs_dataset = format_dict_items(product_type_def_params, **kwargs)["dataset"]
         start_date = kwargs.pop("startTimeFromAscendingNode", None)
@@ -172,11 +196,39 @@ class UsgsApi(Api):
                 max_results=items_per_page,
                 starting_number=(1 + (page - 1) * items_per_page),
             )
-            logger.info(
-                f"Sending search request for {usgs_dataset} with {api_search_kwargs}"
-            )
 
-            results = api.scene_search(usgs_dataset, **api_search_kwargs)
+            # search by id
+            if searched_id := kwargs.get("id"):
+                dataset_filters = api.dataset_filters(usgs_dataset)
+                # ip pattern set as parameter queryable (first element of param conf list)
+                id_pattern = self.config.metadata_mapping["id"][0]
+                # loop on matching dataset_filters until one returns expected results
+                for dataset_filter in dataset_filters["data"]:
+                    if id_pattern in dataset_filter["searchSql"]:
+                        logger.debug(
+                            f"Try using {dataset_filter['searchSql']} dataset filter to search by id on {usgs_dataset}"
+                        )
+                        full_api_search_kwargs = {
+                            "where": {
+                                "filter_id": dataset_filter["id"],
+                                "value": searched_id,
+                            },
+                            **api_search_kwargs,
+                        }
+                        logger.info(
+                            f"Sending search request for {usgs_dataset} with {full_api_search_kwargs}"
+                        )
+                        results = api.scene_search(
+                            usgs_dataset, **full_api_search_kwargs
+                        )
+                        if len(results["data"]["results"]) == 1:
+                            # search by id using this dataset_filter succeeded
+                            break
+            else:
+                logger.info(
+                    f"Sending search request for {usgs_dataset} with {api_search_kwargs}"
+                )
+                results = api.scene_search(usgs_dataset, **api_search_kwargs)
 
             # update results with storage info from download_options()
             results_by_entity_id = {
@@ -227,7 +279,7 @@ class UsgsApi(Api):
                 f"Product type {usgs_dataset} may not exist on USGS EE catalog"
             )
             api.logout()
-            raise RequestError(e)
+            raise RequestError.from_error(e) from e
 
         api.logout()
 
@@ -257,13 +309,13 @@ class UsgsApi(Api):
             )
             progress_callback = ProgressCallback(disable=True)
 
-        outputs_extension = cast(
+        output_extension = cast(
             str,
             self.config.products.get(  # type: ignore
                 product.product_type, self.config.products[GENERIC_PRODUCT_TYPE]  # type: ignore
-            ).get("outputs_extension", ".tar.gz"),
+            ).get("output_extension", ".tar.gz"),
         )
-        kwargs["outputs_extension"] = kwargs.get("outputs_extension", outputs_extension)
+        kwargs["output_extension"] = kwargs.get("output_extension", output_extension)
 
         fs_path, record_filename = self._prepare_download(
             product,
@@ -375,8 +427,8 @@ class UsgsApi(Api):
 
         # Check downloaded file format
         if (
-            kwargs["outputs_extension"] == ".tar.gz" and tarfile.is_tarfile(fs_path)
-        ) or (kwargs["outputs_extension"] == ".zip" and zipfile.is_zipfile(fs_path)):
+            kwargs["output_extension"] == ".tar.gz" and tarfile.is_tarfile(fs_path)
+        ) or (kwargs["output_extension"] == ".zip" and zipfile.is_zipfile(fs_path)):
             product_path = self._finalize(
                 fs_path,
                 progress_callback=progress_callback,
@@ -388,7 +440,7 @@ class UsgsApi(Api):
             logger.info(
                 "Downloaded product detected as a tar File, but was was expected to be a zip file"
             )
-            new_fs_path = fs_path[: fs_path.index(outputs_extension)] + ".tar.gz"
+            new_fs_path = fs_path[: fs_path.index(output_extension)] + ".tar.gz"
             shutil.move(fs_path, new_fs_path)
             product.location = path_to_uri(new_fs_path)
             return new_fs_path
@@ -396,7 +448,7 @@ class UsgsApi(Api):
             logger.info(
                 "Downloaded product detected as a zip File, but was was expected to be a tar file"
             )
-            new_fs_path = fs_path[: fs_path.index(outputs_extension)] + ".zip"
+            new_fs_path = fs_path[: fs_path.index(output_extension)] + ".zip"
             shutil.move(fs_path, new_fs_path)
             product.location = path_to_uri(new_fs_path)
             return new_fs_path
@@ -404,7 +456,7 @@ class UsgsApi(Api):
             logger.warning(
                 "Downloaded product is not a tar or a zip File. Please check its file type before using it"
             )
-            new_fs_path = fs_path[: fs_path.index(outputs_extension)]
+            new_fs_path = fs_path[: fs_path.index(output_extension)]
             shutil.move(fs_path, new_fs_path)
             product.location = path_to_uri(new_fs_path)
             return new_fs_path

eodag/plugins/authentication/aws_auth.py

@@ -30,13 +30,23 @@ if TYPE_CHECKING:
 class AwsAuth(Authentication):
     """AWS authentication plugin
 
-    Authentication will use the first valid method within the following ones:
+    Authentication will use the first valid method within the following ones depending on which
+    parameters are available in the configuration:
 
-    - auth anonymously using no-sign-request
-    - auth using ``aws_profile``
-    - auth using ``aws_access_key_id`` and ``aws_secret_access_key``
-    - auth using current environment (AWS environment variables and/or ``~/aws/*``),
+    * auth anonymously using no-sign-request
+    * auth using ``aws_profile``
+    * auth using ``aws_access_key_id`` and ``aws_secret_access_key``
+      (optionally ``aws_session_token``)
+    * auth using current environment (AWS environment variables and/or ``~/aws/*``),
       will be skipped if AWS credentials are filled in eodag conf
+
+    :param provider: provider name
+    :param config: Authentication plugin configuration:
+
+        * :attr:`~eodag.config.PluginConfig.type` (``str``) (**mandatory**): AwsAuth
+        * :attr:`~eodag.config.PluginConfig.auth_error_code` (``int``) (mandatory for ``creodias_s3``):
+          which error code is returned in case of an authentication error
+
     """
 
     s3_client: S3Client
@@ -45,13 +55,13 @@ class AwsAuth(Authentication):
         super(AwsAuth, self).__init__(provider, config)
         self.aws_access_key_id = None
         self.aws_secret_access_key = None
+        self.aws_session_token = None
         self.profile_name = None
 
     def authenticate(self) -> Dict[str, str]:
         """Authenticate
 
         :returns: dict containing AWS/boto3 non-empty credentials
-        :rtype: dict
         """
         credentials = getattr(self.config, "credentials", {}) or {}
         self.aws_access_key_id = credentials.get(
@@ -60,7 +70,15 @@ class AwsAuth(Authentication):
         self.aws_secret_access_key = credentials.get(
             "aws_secret_access_key", self.aws_secret_access_key
         )
+        self.aws_session_token = credentials.get(
+            "aws_session_token", self.aws_session_token
+        )
         self.profile_name = credentials.get("aws_profile", self.profile_name)
 
-        auth_keys = ["aws_access_key_id", "aws_secret_access_key", "profile_name"]
+        auth_keys = [
+            "aws_access_key_id",
+            "aws_secret_access_key",
+            "aws_session_token",
+            "profile_name",
+        ]
         return {k: getattr(self, k) for k in auth_keys if getattr(self, k)}

eodag/plugins/authentication/base.py

@@ -27,7 +27,16 @@ if TYPE_CHECKING:
 
 
 class Authentication(PluginTopic):
-    """Plugins authentication Base plugin"""
+    """Plugins authentication Base plugin
+
+    :param provider: provider name
+    :param config: Authentication plugin configuration:
+
+        * :attr:`~eodag.config.PluginConfig.matching_url` (``str``): URL pattern to match with search plugin endpoint or
+          download link
+        * :attr:`~eodag.config.PluginConfig.matching_conf` (``Dict[str, Any]``): Part of the search or download plugin
+          configuration that needs authentication and helps identifying it
+    """
 
     def authenticate(self) -> Union[AuthBase, Dict[str, str]]:
         """Authenticate"""

eodag/plugins/authentication/generic.py

@@ -29,7 +29,18 @@ if TYPE_CHECKING:
 
 
 class GenericAuth(Authentication):
-    """GenericAuth authentication plugin"""
+    """GenericAuth authentication plugin (authentication using ``username`` and ``password``)
+
+    The mandatory parameters that have to be added in the eodag config are username and password.
+
+    :param provider: provider name
+    :param config: Authentication plugin configuration:
+
+        * :attr:`~eodag.config.PluginConfig.type` (``str``) (**mandatory**): GenericAuth
+        * :attr:`~eodag.config.PluginConfig.method` (``str``): specifies if digest authentication
+          (``digest``) or basic authentication (``basic``) should be used; default: ``basic``
+
+    """
 
     def authenticate(self) -> AuthBase:
         """Authenticate"""
@@ -48,6 +59,6 @@ class GenericAuth(Authentication):
             )
         else:
             raise MisconfiguredError(
-                f"Cannot authenticate with {self.provider}:",
-                f"method {method} is not supported. Must be one of digest or basic",
+                f"Cannot authenticate with {self.provider}",
+                f"Method {method} is not supported; it must be one of 'digest' or 'basic'.",
             )

eodag/plugins/authentication/header.py

@@ -34,7 +34,14 @@ class HTTPHeaderAuth(Authentication):
     This plugin enables implementation of custom HTTP authentication scheme (other than Basic, Digest, Token
     negotiation et al.) using HTTP headers.
 
-    The plugin is configured as follows in the providers config file::
+    :param provider: provider name
+    :param config: Authentication plugin configuration:
+
+        * :attr:`~eodag.config.PluginConfig.type` (``str``) (**mandatory**): HTTPHeaderAuth
+        * :attr:`~eodag.config.PluginConfig.headers` (``Dict[str, str]``): dictionary containing
+          all keys/value pairs that should be added to the headers
+
+    Below an example for the configuration in the providers config file is shown::
 
         provider:
             ...
@@ -47,9 +54,9 @@ class HTTPHeaderAuth(Authentication):
                 ...
             ...
 
-    As you can see in the sample above, the maintainer of `provider` define the headers that will be used in the
-    authentication process as-is, by giving their names (e.g. `Authorization`) and their value (e.g
-    `"Something {userinput}"`) as regular Python string templates that enable passing in the user input necessary to
+    As you can see in the sample above, the maintainer of ``provider`` define the headers that will be used in the
+    authentication process as-is, by giving their names (e.g. ``Authorization``) and their value (e.g
+    ``"Something {userinput}"``) as regular Python string templates that enable passing in the user input necessary to
     compute its identity. The user input awaited in the header value string must be present in the user config file.
     In the sample above, the plugin await for user credentials to be specified as::
 
@@ -72,6 +79,7 @@ class HTTPHeaderAuth(Authentication):
                     X-Another-Special-Header: "YYY"
                 ...
             ...
+
     """
 
     def authenticate(self) -> HeaderAuth:

eodag/plugins/authentication/keycloak.py

@@ -41,19 +41,38 @@ logger = logging.getLogger("eodag.auth.keycloak")
 class KeycloakOIDCPasswordAuth(OIDCRefreshTokenBase):
     """Authentication plugin using Keycloak and OpenId Connect.
 
-    This plugin request a token and use it through a query-string or a header.
+    This plugin requests a token which is added to a query-string or a header for authentication.
+
+    :param provider: provider name
+    :param config: Authentication plugin configuration:
+
+        * :attr:`~eodag.config.PluginConfig.type` (``str``) (**mandatory**): KeycloakOIDCPasswordAuth
+        * :attr:`~eodag.config.PluginConfig.oidc_config_url` (``str``) (**mandatory**):
+          The url to get the OIDC Provider's endpoints
+        * :attr:`~eodag.config.PluginConfig.client_id` (``str``) (**mandatory**): keycloak client id
+        * :attr:`~eodag.config.PluginConfig.client_secret` (``str``) (**mandatory**): keycloak
+          client secret, set to null if no secret is used
+        * :attr:`~eodag.config.PluginConfig.token_provision` (``str``) (**mandatory**): if the
+          token should be added to the query string (``qs``) or to the header (``header``)
+        * :attr:`~eodag.config.PluginConfig.token_qs_key` (``str``): (**mandatory if token_provision=qs**)
+          key of the param added to the query string
+        * :attr:`~eodag.config.PluginConfig.allowed_audiences` (``List[str]``) (**mandatory**):
+          The allowed audiences that have to be present in the user token.
+        * :attr:`~eodag.config.PluginConfig.auth_error_code` (``int``): which error code is
+          returned in case of an authentication error
+        * :attr:`~eodag.config.PluginConfig.ssl_verify` (``bool``): if the ssl certificates
+          should be verified in the token request; default: ``True``
 
     Using :class:`~eodag.plugins.download.http.HTTPDownload` a download link
-    `http://example.com?foo=bar` will become
-    `http://example.com?foo=bar&my-token=obtained-token` if associated to the following
+    ``http://example.com?foo=bar`` will become
+    ``http://example.com?foo=bar&my-token=obtained-token`` if associated to the following
     configuration::
 
         provider:
             ...
             auth:
                 plugin: KeycloakOIDCPasswordAuth
-                auth_base_uri: 'https://somewhere/auth'
-                realm: 'the-realm'
+                oidc_config_url: 'https://somewhere/auth/realms/realm/.well-known/openid-configuration'
                 client_id: 'SOME_ID'
                 client_secret: '01234-56789'
                 token_provision: qs
@@ -62,15 +81,14 @@ class KeycloakOIDCPasswordAuth(OIDCRefreshTokenBase):
             ...
 
     If configured to send the token through the header, the download request header will
-    be updated with `Authorization: "Bearer obtained-token"` if associated to the
+    be updated with ``Authorization: "Bearer obtained-token"`` if associated to the
     following configuration::
 
         provider:
             ...
             auth:
                 plugin: KeycloakOIDCPasswordAuth
-                auth_base_uri: 'https://somewhere/auth'
-                realm: 'the-realm'
+                oidc_config_url: 'https://somewhere/auth/realms/realm/.well-known/openid-configuration'
                 client_id: 'SOME_ID'
                 client_secret: '01234-56789'
                 token_provision: header
@@ -79,8 +97,12 @@ class KeycloakOIDCPasswordAuth(OIDCRefreshTokenBase):
     """
 
     GRANT_TYPE = "password"
-    TOKEN_URL_TEMPLATE = "{auth_base_uri}/realms/{realm}/protocol/openid-connect/token"
-    REQUIRED_PARAMS = ["auth_base_uri", "client_id", "client_secret", "token_provision"]
+    REQUIRED_PARAMS = [
+        "oidc_config_url",
+        "client_id",
+        "client_secret",
+        "token_provision",
+    ]
 
     def __init__(self, provider: str, config: PluginConfig) -> None:
         super(KeycloakOIDCPasswordAuth, self).__init__(provider, config)
@@ -101,10 +123,9 @@ class KeycloakOIDCPasswordAuth(OIDCRefreshTokenBase):
         Makes authentication request
         """
         self.validate_config_credentials()
-        access_token = self._get_access_token()
-        self.token_info["access_token"] = access_token
+        self._get_access_token()
         return CodeAuthorizedAuth(
-            self.token_info["access_token"],
+            self.access_token,
             self.config.token_provision,
             key=getattr(self.config, "token_qs_key", None),
         )
@@ -117,15 +138,14 @@ class KeycloakOIDCPasswordAuth(OIDCRefreshTokenBase):
             "grant_type": self.GRANT_TYPE,
         }
         credentials = {k: v for k, v in self.config.credentials.items()}
+        ssl_verify = getattr(self.config, "ssl_verify", True)
         try:
             response = self.session.post(
-                self.TOKEN_URL_TEMPLATE.format(
-                    auth_base_uri=self.config.auth_base_uri.rstrip("/"),
-                    realm=self.config.realm,
-                ),
+                self.token_endpoint,
                 data=dict(req_data, **credentials),
                 headers=USER_AGENT,
                 timeout=HTTP_REQ_TIMEOUT,
+                verify=ssl_verify,
             )
             response.raise_for_status()
         except requests.exceptions.Timeout as exc:
@@ -140,17 +160,16 @@ class KeycloakOIDCPasswordAuth(OIDCRefreshTokenBase):
             "client_id": self.config.client_id,
             "client_secret": self.config.client_secret,
             "grant_type": "refresh_token",
-            "refresh_token": self.token_info["refresh_token"],
+            "refresh_token": self.refresh_token,
         }
+        ssl_verify = getattr(self.config, "ssl_verify", True)
         try:
             response = self.session.post(
-                self.TOKEN_URL_TEMPLATE.format(
-                    auth_base_uri=self.config.auth_base_uri.rstrip("/"),
-                    realm=self.config.realm,
-                ),
+                self.token_endpoint,
                 data=req_data,
                 headers=USER_AGENT,
                 timeout=HTTP_REQ_TIMEOUT,
+                verify=ssl_verify,
             )
             response.raise_for_status()
         except requests.RequestException as e:

eodag/plugins/authentication/oauth.py

@@ -26,7 +26,17 @@ if TYPE_CHECKING:
 
 
 class OAuth(Authentication):
-    """OAuth authentication plugin"""
+    """OAuth authentication plugin
+
+    The mandatory parameters that have to be added in the eodag config are ``aws_access_key_id``
+    and ``aws_secret_access_key``.
+
+    :param provider: provider name
+    :param config: Authentication plugin configuration:
+
+        * :attr:`~eodag.config.PluginConfig.type` (``str``) (**mandatory**): OAuth
+
+    """
 
     def __init__(self, provider: str, config: PluginConfig) -> None:
         super(OAuth, self).__init__(provider, config)