eodag 3.8.0__py3-none-any.whl → 3.9.0__py3-none-any.whl

eodag/plugins/authentication/aws_auth.py

@@ -17,29 +17,72 @@
 # limitations under the License.
 from __future__ import annotations
 
-from typing import TYPE_CHECKING, Optional, cast
+import logging
+from typing import TYPE_CHECKING, Any, Optional, cast
+
+import boto3
+from botocore.exceptions import ClientError, ProfileNotFound
+from botocore.handlers import disable_signing
 
 from eodag.plugins.authentication.base import Authentication
 from eodag.types import S3SessionKwargs
+from eodag.utils.exceptions import AuthenticationError
 
 if TYPE_CHECKING:
-    from mypy_boto3_s3.client import S3Client
+    from mypy_boto3_s3 import S3Client, S3ServiceResource
+    from mypy_boto3_s3.service_resource import BucketObjectsCollection
 
     from eodag.config import PluginConfig
 
 
+logger = logging.getLogger("eodag.download.aws_auth")
+
+AWS_AUTH_ERROR_MESSAGES = [
+    "AccessDenied",
+    "InvalidAccessKeyId",
+    "SignatureDoesNotMatch",
+    "InvalidRequest",
+]
+
+
+def raise_if_auth_error(exception: ClientError, provider: str) -> None:
+    """Raises an error if given exception is an authentication error"""
+    err = cast(dict[str, str], exception.response["Error"])
+    if err["Code"] in AWS_AUTH_ERROR_MESSAGES and "key" in err["Message"].lower():
+        raise AuthenticationError(
+            f"Please check your credentials for {provider}.",
+            f"HTTP Error {exception.response['ResponseMetadata']['HTTPStatusCode']} returned.",
+            err["Code"] + ": " + err["Message"],
+        )
+
+
+def create_s3_session(**kwargs) -> boto3.Session:
+    """create s3 session based on available credentials
+
+    :param kwargs: keyword arguments containing credentials
+    :returns: boto3 Session
+    """
+    try:
+        s3_session = boto3.Session(**kwargs)
+    except ProfileNotFound:
+        raise AuthenticationError(
+            f"AWS profile {kwargs['profile_name']} not found, please check your credentials configuration"
+        )
+    return s3_session
+
+
 class AwsAuth(Authentication):
     """AWS authentication plugin
 
-    Authentication will use the first valid method within the following ones depending on which
-    parameters are available in the configuration:
+    The authentication method will be chosen depending on which parameters are available in the configuration:
 
-    * auth anonymously using no-sign-request
-    * auth using ``aws_profile``
-    * auth using ``aws_access_key_id`` and ``aws_secret_access_key``
-      (optionally ``aws_session_token``)
-    * auth using current environment (AWS environment variables and/or ``~/aws/*``),
-      will be skipped if AWS credentials are filled in eodag conf
+    * auth using ``profile_name`` (if credentials are given and contain ``aws_profile``)
+    * auth using ``aws_access_key_id``, ``aws_secret_access_key`` and optionally ``aws_session_token``
+      (if credentials are given but no ``aws_profile``)
+    * auth using current environment - AWS environment variables and/or ``~/.aws/*``
+      (if no credentials are given in config)
+    * auth anonymously using no-sign-request if no credentials are given in config and
+      auth using current environment failed
 
     :param provider: provider name
     :param config: Authentication plugin configuration:
@@ -47,41 +90,189 @@ class AwsAuth(Authentication):
         * :attr:`~eodag.config.PluginConfig.type` (``str``) (**mandatory**): AwsAuth
         * :attr:`~eodag.config.PluginConfig.auth_error_code` (``int``) (mandatory for ``creodias_s3``):
           which error code is returned in case of an authentication error
+        * :attr:`~eodag.config.PluginConfig.s3_endpoint` (``str``): s3 endpoint url
+        * :attr:`~eodag.config.PluginConfig.requester_pays` (``bool``): whether download is done
+          from a requester-pays bucket or not; default: ``False``
 
     """
 
-    s3_client: S3Client
-
     def __init__(self, provider: str, config: PluginConfig) -> None:
         super(AwsAuth, self).__init__(provider, config)
-        self.aws_access_key_id: Optional[str] = None
-        self.aws_secret_access_key: Optional[str] = None
-        self.aws_session_token: Optional[str] = None
-        self.profile_name: Optional[str] = None
+        self.s3_session: Optional[boto3.Session] = None
+        self.s3_resource: Optional[S3ServiceResource] = None
+        # set default for requester_pays if not given
+        self.config.__dict__.setdefault("requester_pays", False)
+
+    def _create_s3_session_from_credentials(self) -> boto3.Session:
+        credentials = getattr(self.config, "credentials", {}) or {}
+        if "aws_profile" in credentials:
+            return create_s3_session(profile_name=credentials["aws_profile"])
+        # auth using aws keys
+        elif credentials:
+            s3_session_kwargs: S3SessionKwargs = {
+                "aws_access_key_id": credentials["aws_access_key_id"],
+                "aws_secret_access_key": credentials["aws_secret_access_key"],
+            }
+            if credentials.get("aws_session_token"):
+                s3_session_kwargs["aws_session_token"] = credentials[
+                    "aws_session_token"
+                ]
+            return create_s3_session(**s3_session_kwargs)
+        else:
+            # auth using env variables or ~/.aws
+            return create_s3_session()
+
+    def _create_s3_resource(self) -> S3ServiceResource:
+        """create s3 resource based on s3 session"""
+        if not self.s3_session:
+            self.s3_session = self._create_s3_session_from_credentials()
+        endpoint_url = getattr(self.config, "s3_endpoint", None)
+        if self.s3_session.get_credentials():
+            return self.s3_session.resource(
+                service_name="s3",
+                endpoint_url=endpoint_url,
+            )
+        # could not auth using credentials: use no-sign-request strategy
+        s3_resource = boto3.resource(service_name="s3", endpoint_url=endpoint_url)
+        s3_resource.meta.client.meta.events.register(
+            "choose-signer.s3.*", disable_signing
+        )
+        return s3_resource
+
+    def get_s3_client(self) -> S3Client:
+        """Get S3 client from S3 resource
 
-    def authenticate(self) -> S3SessionKwargs:
+        :returns: boto3 client
+        """
+        if not self.s3_resource:
+            self.s3_resource = self._create_s3_resource()
+        return self.s3_resource.meta.client
+
+    def authenticate(self) -> S3ServiceResource:
         """Authenticate
 
-        :returns: dict containing AWS/boto3 non-empty credentials
+        :returns: S3 Resource created based on an S3 session
         """
-        credentials = getattr(self.config, "credentials", {}) or {}
-        self.aws_access_key_id = credentials.get(
-            "aws_access_key_id", self.aws_access_key_id
-        )
-        self.aws_secret_access_key = credentials.get(
-            "aws_secret_access_key", self.aws_secret_access_key
-        )
-        self.aws_session_token = credentials.get(
-            "aws_session_token", self.aws_session_token
+        self.s3_resource = self._create_s3_resource()
+        return self.s3_resource
+
+    def _get_authenticated_objects(
+        self, bucket_name: str, prefix: str
+    ) -> BucketObjectsCollection:
+        """Get boto3 authenticated objects for the given bucket
+
+        :param bucket_name: Bucket containg objects
+        :param prefix: Prefix used to filter objects
+        :returns: The boto3 authenticated objects
+        """
+        if not self.s3_resource:
+            self.s3_resource = self._create_s3_resource()
+        try:
+            if self.config.requester_pays:
+                objects = self.s3_resource.Bucket(bucket_name).objects.filter(
+                    RequestPayer="requester"
+                )
+            else:
+                objects = self.s3_resource.Bucket(bucket_name).objects
+            list(objects.filter(Prefix=prefix).limit(1))
+            if objects:
+                logger.debug(
+                    "Authentication for bucket %s succeeded; returning available objects",
+                    bucket_name,
+                )
+                return objects
+        except ClientError as e:
+            if e.response.get("Error", {}).get("Code", {}) in AWS_AUTH_ERROR_MESSAGES:
+                pass
+            else:
+                raise e
+        logger.debug(
+            "Authentication for bucket %s failed, please check the credentials",
+            bucket_name,
         )
-        self.profile_name = credentials.get("aws_profile", self.profile_name)
-
-        auth_dict = cast(
-            S3SessionKwargs,
-            {
-                k: getattr(self, k)
-                for k in S3SessionKwargs.__annotations__
-                if getattr(self, k, None)
-            },
+
+        raise AuthenticationError(
+            "Unable do authenticate on s3://%s using credendials configuration"
+            % bucket_name
         )
-        return auth_dict
+
+    def authenticate_objects(
+        self,
+        bucket_names_and_prefixes: list[tuple[str, Optional[str]]],
+    ) -> dict[str, BucketObjectsCollection]:
+        """
+        Authenticates with s3 and retrieves the available objects
+
+        :param bucket_names_and_prefixes: list of bucket names and corresponding path prefixes
+        :raises AuthenticationError: authentication is not possible
+        :return: authenticated objects per bucket
+        """
+
+        authenticated_objects: dict[str, Any] = {}
+        auth_error_messages: set[str] = set()
+        for _, pack in enumerate(bucket_names_and_prefixes):
+
+            bucket_name, prefix = pack
+            if not prefix:
+                continue
+            if bucket_name not in authenticated_objects:
+                # get Prefixes longest common base path
+                common_prefix = ""
+                prefix_split = prefix.split("/")
+                prefixes_in_bucket = len(
+                    [p for b, p in bucket_names_and_prefixes if b == bucket_name]
+                )
+                for i in range(1, len(prefix_split)):
+                    common_prefix = "/".join(prefix_split[0:i])
+                    if (
+                        len(
+                            [
+                                p
+                                for b, p in bucket_names_and_prefixes
+                                if p and b == bucket_name and common_prefix in p
+                            ]
+                        )
+                        < prefixes_in_bucket
+                    ):
+                        common_prefix = "/".join(prefix_split[0 : i - 1])
+                        break
+                try:
+                    # connect to aws s3 and get bucket auhenticated objects
+                    authenticated_objects[
+                        bucket_name
+                    ] = self._get_authenticated_objects(bucket_name, common_prefix)
+
+                except AuthenticationError as e:
+                    logger.warning("Unexpected error: %s" % e)
+                    logger.warning("Skipping %s/%s" % (bucket_name, prefix))
+                    auth_error_messages.add(str(e))
+                except ClientError as e:
+                    raise_if_auth_error(e, self.provider)
+                    logger.warning("Unexpected error: %s" % e)
+                    logger.warning("Skipping %s/%s" % (bucket_name, prefix))
+                    auth_error_messages.add(str(e))
+
+        # could not auth on any bucket
+        if not authenticated_objects:
+            raise AuthenticationError(", ".join(auth_error_messages))
+        return authenticated_objects
+
+    def get_rio_env(self) -> dict[str, Any]:
+        """Get rasterio environment variables needed for data access authentication.
+
+        :returns: The rasterio environement variables
+        """
+        rio_env_kwargs = {}
+        if endpoint_url := getattr(self.config, "s3_endpoint", None):
+            rio_env_kwargs["endpoint_url"] = endpoint_url.split("://")[-1]
+
+        if self.s3_session is None:
+            self.authenticate()
+
+        if self.config.requester_pays:
+            rio_env_kwargs["requester_pays"] = True
+
+        return {
+            "session": self.s3_session,
+            **rio_env_kwargs,
+        }

eodag/plugins/authentication/base.py

@@ -17,12 +17,13 @@
 # limitations under the License.
 from __future__ import annotations
 
-from typing import TYPE_CHECKING, Union
+from typing import TYPE_CHECKING, Any, Optional, Union
 
 from eodag.plugins.base import PluginTopic
 from eodag.utils.exceptions import MisconfiguredError
 
 if TYPE_CHECKING:
+    from mypy_boto3_s3 import S3ServiceResource
     from requests.auth import AuthBase
 
     from eodag.types import S3SessionKwargs
@@ -40,7 +41,7 @@ class Authentication(PluginTopic):
           configuration that needs authentication and helps identifying it
     """
 
-    def authenticate(self) -> Union[AuthBase, S3SessionKwargs]:
+    def authenticate(self) -> Union[AuthBase, S3SessionKwargs, S3ServiceResource]:
         """Authenticate"""
         raise NotImplementedError
 
@@ -70,3 +71,12 @@ class Authentication(PluginTopic):
                     self.provider, ", ".join(missing_credentials)
                 )
             )
+
+    def authenticate_objects(
+        self,
+        bucket_names_and_prefixes: list[tuple[str, Optional[str]]],
+    ) -> dict[str, Any]:
+        """
+        Authenticates with s3 and retrieves the available objects
+        """
+        raise NotImplementedError

eodag/plugins/authentication/oauth.py

@@ -20,12 +20,17 @@ from __future__ import annotations
 from typing import TYPE_CHECKING, Optional
 
 from eodag.plugins.authentication.base import Authentication
+from eodag.utils import _deprecated
 
 if TYPE_CHECKING:
     from eodag.config import PluginConfig
     from eodag.types import S3SessionKwargs
 
 
+@_deprecated(
+    reason="Plugin was used to authenticate using S3 credentials, use AwsAuth instead",
+    version="3.8.1",
+)
 class OAuth(Authentication):
     """OAuth authentication plugin
 

eodag/plugins/authentication/sas_auth.py

@@ -18,6 +18,7 @@
 from __future__ import annotations
 
 import logging
+import re
 from json import JSONDecodeError
 from typing import TYPE_CHECKING, Optional
 
@@ -29,6 +30,8 @@ from eodag.utils import HTTP_REQ_TIMEOUT, USER_AGENT, deepcopy, format_dict_item
 from eodag.utils.exceptions import AuthenticationError, TimeOutError
 
 if TYPE_CHECKING:
+    from typing import Pattern
+
     from requests import PreparedRequest
 
 
@@ -44,15 +47,24 @@ class RequestsSASAuth(AuthBase):
         signed_url_key: str,
         headers: Optional[dict[str, str]] = None,
         ssl_verify: bool = True,
+        matching_url: Optional[Pattern[str]] = None,
     ) -> None:
         self.auth_uri = auth_uri
         self.signed_url_key = signed_url_key
         self.headers = headers
         self.signed_urls: dict[str, str] = {}
         self.ssl_verify = ssl_verify
+        self.matching_url = matching_url
 
     def __call__(self, request: PreparedRequest) -> PreparedRequest:
         """Perform the actual authentication"""
+        # if matching_url is set, check if request.url matches
+        if (
+            self.matching_url
+            and request.url
+            and not self.matching_url.match(request.url)
+        ):
+            return request
 
         # update headers
         if self.headers and isinstance(self.headers, dict):
@@ -118,6 +130,8 @@ class SASAuth(Authentication):
         # update headers with subscription key if exists
         apikey = getattr(self.config, "credentials", {}).get("apikey")
         ssl_verify = getattr(self.config, "ssl_verify", True)
+        if matching_url := getattr(self.config, "matching_url", None):
+            matching_url = re.compile(matching_url)
         if apikey:
             headers_update = format_dict_items(self.config.headers, apikey=apikey)
             headers.update(headers_update)
@@ -127,4 +141,5 @@ class SASAuth(Authentication):
             signed_url_key=self.config.signed_url_key,
             headers=headers,
             ssl_verify=ssl_verify,
+            matching_url=matching_url,
         )

eodag/plugins/base.py

@@ -65,9 +65,10 @@ class PluginTopic(metaclass=EODAGPluginMount):
         self.provider = provider
 
     def __repr__(self) -> str:
+        config = getattr(self, "config", None)
         return "{}(provider={}, priority={}, topic={})".format(
             self.__class__.__name__,
-            self.provider,
-            self.config.priority,
+            getattr(self, "provider", ""),
+            config.priority if config else "",
             self.__class__.mro()[-3].__name__,
         )