eodag 2.12.1__py3-none-any.whl → 3.0.0b1__py3-none-any.whl

eodag/plugins/apis/usgs.py

@@ -18,10 +18,11 @@
 from __future__ import annotations
 
 import logging
+import os
 import shutil
 import tarfile
 import zipfile
-from typing import TYPE_CHECKING, Any, Dict, List, Optional, Tuple, cast
+from typing import TYPE_CHECKING, Any, Dict, List, Optional, Tuple, Union, cast
 
 import requests
 from jsonpath_ng.ext import parse
@@ -35,7 +36,7 @@ from eodag.api.product.metadata_mapping import (
     properties_from_json,
 )
 from eodag.plugins.apis.base import Api
-from eodag.plugins.download.base import Download
+from eodag.plugins.search import PreparedSearch
 from eodag.utils import (
     DEFAULT_DOWNLOAD_TIMEOUT,
     DEFAULT_DOWNLOAD_WAIT,
@@ -52,17 +53,21 @@ from eodag.utils.exceptions import (
     NoMatchingProductType,
     NotAvailableError,
     RequestError,
+    ValidationError,
 )
 
 if TYPE_CHECKING:
+    from requests.auth import AuthBase
+
     from eodag.api.search_result import SearchResult
     from eodag.config import PluginConfig
-    from eodag.utils import DownloadedCallback
+    from eodag.types.download_args import DownloadConf
+    from eodag.utils import DownloadedCallback, Unpack
 
 logger = logging.getLogger("eodag.apis.usgs")
 
 
-class UsgsApi(Download, Api):
+class UsgsApi(Api):
     """A plugin that enables to query and download data on the USGS catalogues"""
 
     def __init__(self, provider: str, config: PluginConfig) -> None:
@@ -71,7 +76,7 @@ class UsgsApi(Download, Api):
         # Same method as in base.py, Search.__init__()
         # Prepare the metadata mapping
         # Do a shallow copy, the structure is flat enough for this to be sufficient
-        metas = DEFAULT_METADATA_MAPPING.copy()
+        metas: Dict[str, Any] = DEFAULT_METADATA_MAPPING.copy()
         # Update the defaults with the mapping value. This will add any new key
         # added by the provider mapping that is not in the default metadata.
         metas.update(self.config.metadata_mapping)
@@ -98,25 +103,33 @@ class UsgsApi(Download, Api):
             except USGSAuthExpiredError:
                 api.logout()
                 continue
-            except USGSError:
-                raise AuthenticationError(
-                    "Please check your USGS credentials."
-                ) from None
+            except USGSError as e:
+                if i == 0:
+                    # `.usgs` API file key might be obsolete
+                    # Remove it and try again
+                    os.remove(api.TMPFILE)
+                    continue
+                raise AuthenticationError("Please check your USGS credentials.") from e
 
     def query(
         self,
-        product_type: Optional[str] = None,
-        items_per_page: int = DEFAULT_ITEMS_PER_PAGE,
-        page: int = DEFAULT_PAGE,
-        count: bool = True,
+        prep: PreparedSearch = PreparedSearch(),
         **kwargs: Any,
     ) -> Tuple[List[EOProduct], Optional[int]]:
         """Search for data on USGS catalogues"""
+        page = prep.page if prep.page is not None else DEFAULT_PAGE
+        items_per_page = (
+            prep.items_per_page
+            if prep.items_per_page is not None
+            else DEFAULT_ITEMS_PER_PAGE
+        )
         product_type = kwargs.get("productType")
         if product_type is None:
             raise NoMatchingProductType(
                 "Cannot search on USGS without productType specified"
             )
+        if kwargs.get("sortBy"):
+            raise ValidationError("USGS does not support sorting feature")
 
         self.authenticate()
 
@@ -230,11 +243,11 @@ class UsgsApi(Download, Api):
     def download(
         self,
         product: EOProduct,
-        auth: Optional[PluginConfig] = None,
+        auth: Optional[Union[AuthBase, Dict[str, str]]] = None,
         progress_callback: Optional[ProgressCallback] = None,
         wait: int = DEFAULT_DOWNLOAD_WAIT,
         timeout: int = DEFAULT_DOWNLOAD_TIMEOUT,
-        **kwargs: Any,
+        **kwargs: Unpack[DownloadConf],
     ) -> Optional[str]:
         """Download data from USGS catalogues"""
 
@@ -250,11 +263,11 @@ class UsgsApi(Download, Api):
                 product.product_type, self.config.products[GENERIC_PRODUCT_TYPE]  # type: ignore
             ).get("outputs_extension", ".tar.gz"),
         )
+        kwargs["outputs_extension"] = kwargs.get("outputs_extension", outputs_extension)
 
         fs_path, record_filename = self._prepare_download(
             product,
             progress_callback=progress_callback,
-            outputs_extension=outputs_extension,
             **kwargs,
         )
         if not fs_path or not record_filename:
@@ -308,13 +321,14 @@ class UsgsApi(Download, Api):
         req_url = req_urls[0]
         progress_callback.reset()
         logger.debug(f"Downloading {req_url}")
+        ssl_verify = getattr(self.config, "ssl_verify", True)
 
         @self._download_retry(product, wait, timeout)
         def download_request(
             product: EOProduct,
             fs_path: str,
             progress_callback: ProgressCallback,
-            **kwargs: Any,
+            **kwargs: Unpack[DownloadConf],
         ) -> None:
             try:
                 with requests.get(
@@ -322,6 +336,7 @@ class UsgsApi(Download, Api):
                     stream=True,
                     headers=USER_AGENT,
                     timeout=wait * 60,
+                    verify=ssl_verify,
                 ) as stream:
                     try:
                         stream.raise_for_status()
@@ -334,7 +349,9 @@ class UsgsApi(Download, Api):
                             error_message = str(e)
                         raise NotAvailableError(error_message)
                     else:
-                        stream_size = int(stream.headers.get("content-length", 0))
+                        stream_size = (
+                            int(stream.headers.get("content-length", 0)) or None
+                        )
                         progress_callback.reset(total=stream_size)
                         with open(fs_path, "wb") as fhandle:
                             for chunk in stream.iter_content(chunk_size=64 * 1024):
@@ -357,13 +374,12 @@ class UsgsApi(Download, Api):
         api.logout()
 
         # Check downloaded file format
-        if (outputs_extension == ".tar.gz" and tarfile.is_tarfile(fs_path)) or (
-            outputs_extension == ".zip" and zipfile.is_zipfile(fs_path)
-        ):
+        if (
+            kwargs["outputs_extension"] == ".tar.gz" and tarfile.is_tarfile(fs_path)
+        ) or (kwargs["outputs_extension"] == ".zip" and zipfile.is_zipfile(fs_path)):
             product_path = self._finalize(
                 fs_path,
                 progress_callback=progress_callback,
-                outputs_extension=outputs_extension,
                 **kwargs,
             )
             product.location = path_to_uri(product_path)
@@ -396,12 +412,12 @@ class UsgsApi(Download, Api):
     def download_all(
         self,
         products: SearchResult,
-        auth: Optional[PluginConfig] = None,
+        auth: Optional[Union[AuthBase, Dict[str, str]]] = None,
         downloaded_callback: Optional[DownloadedCallback] = None,
         progress_callback: Optional[ProgressCallback] = None,
         wait: int = DEFAULT_DOWNLOAD_WAIT,
         timeout: int = DEFAULT_DOWNLOAD_TIMEOUT,
-        **kwargs: Any,
+        **kwargs: Unpack[DownloadConf],
     ) -> List[str]:
         """
         Download all using parent (base plugin) method

eodag/plugins/authentication/aws_auth.py

@@ -22,7 +22,7 @@ from typing import TYPE_CHECKING, Dict
 from eodag.plugins.authentication.base import Authentication
 
 if TYPE_CHECKING:
-    from botocore.client import S3
+    from mypy_boto3_s3.client import S3Client
 
     from eodag.config import PluginConfig
 
@@ -39,7 +39,7 @@ class AwsAuth(Authentication):
       will be skipped if AWS credentials are filled in eodag conf
     """
 
-    s3_client: S3
+    s3_client: S3Client
 
     def __init__(self, provider: str, config: PluginConfig) -> None:
         super(AwsAuth, self).__init__(provider, config)

eodag/plugins/authentication/header.py

@@ -22,6 +22,7 @@ from typing import TYPE_CHECKING, Dict
 from requests.auth import AuthBase
 
 from eodag.plugins.authentication import Authentication
+from eodag.utils.exceptions import MisconfiguredError
 
 if TYPE_CHECKING:
     from requests import PreparedRequest
@@ -58,16 +59,40 @@ class HTTPHeaderAuth(Authentication):
                 oh-my-another-user-input: YYY
 
     Expect an undefined behaviour if you use empty braces in header value strings.
+
+    The plugin also accepts headers to be passed directly through credentials::
+
+        provider:
+            ...
+            auth:
+                plugin: HTTPHeaderAuth
+                credentials:
+                    Authorization: "Something XXX"
+                    X-Special-Header: "Fixed value"
+                    X-Another-Special-Header: "YYY"
+                ...
+            ...
     """
 
-    def authenticate(self) -> AuthBase:
+    def authenticate(self) -> HeaderAuth:
         """Authenticate"""
         self.validate_config_credentials()
-        headers = {
-            header: value.format(**self.config.credentials)
-            for header, value in self.config.headers.items()
-        }
-        return HeaderAuth(headers)
+        try:
+            headers = (
+                {
+                    header: value.format(**self.config.credentials)
+                    for header, value in self.config.headers.items()
+                }
+                if getattr(self.config, "headers", None)
+                else self.config.credentials
+            )
+            return HeaderAuth(headers)
+        except KeyError as e:
+            raise MisconfiguredError(
+                "The following credentials are missing for provider {}: {}".format(
+                    self.provider, ", ".join(e.args)
+                )
+            )
 
 
 class HeaderAuth(AuthBase):

eodag/plugins/authentication/keycloak.py

@@ -18,15 +18,16 @@
 from __future__ import annotations
 
 import logging
-from datetime import datetime
-from typing import TYPE_CHECKING, Dict, Union
+from typing import TYPE_CHECKING, Any, Dict
 
 import requests
 
-from eodag.plugins.authentication import Authentication
-from eodag.plugins.authentication.openid_connect import CodeAuthorizedAuth
+from eodag.plugins.authentication.openid_connect import (
+    CodeAuthorizedAuth,
+    OIDCRefreshTokenBase,
+)
 from eodag.utils import HTTP_REQ_TIMEOUT, USER_AGENT
-from eodag.utils.exceptions import AuthenticationError, MisconfiguredError
+from eodag.utils.exceptions import MisconfiguredError, TimeOutError
 
 if TYPE_CHECKING:
     from requests.auth import AuthBase
@@ -37,7 +38,7 @@ if TYPE_CHECKING:
 logger = logging.getLogger("eodag.auth.keycloak")
 
 
-class KeycloakOIDCPasswordAuth(Authentication):
+class KeycloakOIDCPasswordAuth(OIDCRefreshTokenBase):
     """Authentication plugin using Keycloak and OpenId Connect.
 
     This plugin request a token and use it through a query-string or a header.
@@ -80,13 +81,9 @@ class KeycloakOIDCPasswordAuth(Authentication):
     GRANT_TYPE = "password"
     TOKEN_URL_TEMPLATE = "{auth_base_uri}/realms/{realm}/protocol/openid-connect/token"
     REQUIRED_PARAMS = ["auth_base_uri", "client_id", "client_secret", "token_provision"]
-    # already retrieved token store, to be used if authenticate() fails (OTP use-case)
-    retrieved_token: str = ""
-    token_info: Dict[str, Union[str, datetime]] = {}
 
     def __init__(self, provider: str, config: PluginConfig) -> None:
         super(KeycloakOIDCPasswordAuth, self).__init__(provider, config)
-        self.session = requests.Session()
 
     def validate_config_credentials(self) -> None:
         """Validate configured credentials"""
@@ -105,51 +102,14 @@ class KeycloakOIDCPasswordAuth(Authentication):
         """
         self.validate_config_credentials()
         access_token = self._get_access_token()
-        self.retrieved_token = access_token
+        self.token_info["access_token"] = access_token
         return CodeAuthorizedAuth(
-            self.retrieved_token,
+            self.token_info["access_token"],
             self.config.token_provision,
             key=getattr(self.config, "token_qs_key", None),
         )
 
-    def _get_access_token(self) -> str:
-        current_time = datetime.now()
-        if (
-            not self.token_info
-            or (
-                "refresh_token" in self.token_info
-                and (current_time - self.token_info["token_time"]).seconds
-                >= self.token_info["refresh_token_expiration"]
-            )
-            or (
-                "refresh_token" not in self.token_info
-                and (current_time - self.token_info["token_time"]).seconds
-                >= self.token_info["access_token_expiration"]
-            )
-        ):
-            # Request new TOKEN on first attempt or if token expired
-            res = self._request_new_token()
-            self.token_info["token_time"] = current_time
-            self.token_info["access_token_expiration"] = res["expires_in"]
-            if "refresh_token" in res:
-                self.token_info["refresh_time"] = current_time
-                self.token_info["refresh_token_expiration"] = res["refresh_expires_in"]
-                self.token_info["refresh_token"] = res["refresh_token"]
-            return res["access_token"]
-        elif (
-            "refresh_token" in self.token_info
-            and (current_time - self.token_info["refresh_time"]).seconds
-            >= self.token_info["access_token_expiration"]
-        ):
-            # Use refresh token
-            res = self._get_token_with_refresh_token()
-            self.token_info["refresh_token"] = res["refresh_token"]
-            self.token_info["refresh_time"] = current_time
-            return res["access_token"]
-        logger.debug("using already retrieved access token")
-        return self.retrieved_token
-
-    def _request_new_token(self) -> Dict[str, str]:
+    def _request_new_token(self) -> Dict[str, Any]:
         logger.debug("fetching new access token")
         req_data = {
             "client_id": self.config.client_id,
@@ -168,41 +128,10 @@ class KeycloakOIDCPasswordAuth(Authentication):
                 timeout=HTTP_REQ_TIMEOUT,
             )
             response.raise_for_status()
+        except requests.exceptions.Timeout as exc:
+            raise TimeOutError(exc, timeout=HTTP_REQ_TIMEOUT) from exc
         except requests.RequestException as e:
-            if self.retrieved_token:
-                # try using already retrieved token if authenticate() fails (OTP use-case)
-                if "access_token_expiration" in self.token_info:
-                    return {
-                        "access_token": self.retrieved_token,
-                        "expires_in": self.token_info["access_token_expiration"],
-                    }
-                else:
-                    return {"access_token": self.retrieved_token, "expires_in": 0}
-            response_text = getattr(e.response, "text", "").strip()
-            # check if error is identified as auth_error in provider conf
-            auth_errors = getattr(self.config, "auth_error_code", [None])
-            if not isinstance(auth_errors, list):
-                auth_errors = [auth_errors]
-            if (
-                hasattr(e.response, "status_code")
-                and e.response.status_code in auth_errors
-            ):
-                raise AuthenticationError(
-                    "HTTP Error %s returned, %s\nPlease check your credentials for %s"
-                    % (e.response.status_code, response_text, self.provider)
-                )
-            # other error
-            else:
-                import traceback as tb
-
-                logger.error(
-                    f"Provider {self.provider} returned {e.response.status_code}: {response_text}"
-                )
-                raise AuthenticationError(
-                    "Something went wrong while trying to get access token:\n{}".format(
-                        tb.format_exc()
-                    )
-                )
+            return self._request_new_token_error(e)
         return response.json()
 
     def _get_token_with_refresh_token(self) -> Dict[str, str]:

eodag/plugins/authentication/oauth.py

@@ -17,7 +17,7 @@
 # limitations under the License.
 from __future__ import annotations
 
-from typing import TYPE_CHECKING, Dict
+from typing import TYPE_CHECKING, Dict, Optional
 
 from eodag.plugins.authentication.base import Authentication
 
@@ -30,8 +30,8 @@ class OAuth(Authentication):
 
     def __init__(self, provider: str, config: PluginConfig) -> None:
         super(OAuth, self).__init__(provider, config)
-        self.access_key = None
-        self.secret_key = None
+        self.access_key: Optional[str] = None
+        self.secret_key: Optional[str] = None
 
     def authenticate(self) -> Dict[str, str]:
         """Authenticate"""