eodag 3.8.1__py3-none-any.whl → 3.9.0__py3-none-any.whl

eodag/api/core.py

@@ -74,6 +74,7 @@ from eodag.utils import (
     string_to_jsonpath,
     uri_to_path,
 )
+from eodag.utils.dates import rfc3339_str_to_datetime
 from eodag.utils.env import is_env_var_true
 from eodag.utils.exceptions import (
     AuthenticationError,
@@ -84,7 +85,6 @@ from eodag.utils.exceptions import (
     UnsupportedProvider,
 )
 from eodag.utils.free_text_search import compile_free_text_query
-from eodag.utils.rest import rfc3339_str_to_datetime
 from eodag.utils.stac_reader import fetch_stac_items
 
 if TYPE_CHECKING:

eodag/api/product/drivers/generic.py

@@ -33,7 +33,11 @@ class GenericDriver(DatasetDriver):
         # data
         {
             "pattern": re.compile(
-                r"^(?:.*[/\\])?([^/\\]+)(\.jp2|\.tiff?|\.dat|\.nc|\.grib2?)$",
+                (
+                    r"^(?:.*[/\\])?([^/\\]+)"
+                    r"(\.jp2|\.tiff?|\.dat|\.nc|\.grib2?|"
+                    r"\.zarr|\.nat|\.covjson|\.parquet|\.zip|\.tar|\.gz)$"
+                ),
                 re.IGNORECASE,
             ),
             "roles": ["data"],

eodag/api/product/metadata_mapping.py

@@ -47,7 +47,6 @@ from eodag.utils import (
     dict_items_recursive_apply,
     format_string,
     get_geometry_from_various,
-    get_timestamp,
     items_recursive_apply,
     nested_pairs2dict,
     remove_str_array_quotes,
@@ -55,6 +54,7 @@ from eodag.utils import (
     string_to_jsonpath,
     update_nested_dict,
 )
+from eodag.utils.dates import get_timestamp
 from eodag.utils.exceptions import ValidationError
 
 if TYPE_CHECKING:
@@ -174,6 +174,7 @@ def format_metadata(search_param: str, *args: Any, **kwargs: Any) -> str:
         - ``slice_str``: slice a string (equivalent to s[start, end, step])
         - ``to_lower``: Convert a string to lowercase
         - ``to_upper``: Convert a string to uppercase
+        - ``to_title``: Convert a string to title case
         - ``fake_l2a_title_from_l1c``: used to generate SAFE format metadata for data from AWS
         - ``s2msil2a_title_to_aws_productinfo``: used to generate SAFE format metadata for data from AWS
         - ``split_cop_dem_id``: get the bbox by splitting the product id
@@ -182,6 +183,8 @@ def format_metadata(search_param: str, *args: Any, **kwargs: Any) -> str:
         - ``get_ecmwf_time``: get the time of a datetime string in the ECMWF format
         - ``sanitize``: sanitize string
         - ``ceda_collection_name``: generate a CEDA collection name from a string
+        - ``convert_dict_filter_and_sub``: filter dict items using jsonpath and then apply recursive_sub_str
+        - ``convert_from_alternate``: update assets using given alternate
 
     :param search_param: The string to be formatted
     :param args: (optional) Additional arguments to use in the formatting process
@@ -529,6 +532,35 @@ def format_metadata(search_param: str, *args: Any, **kwargs: Any) -> str:
             old, new = ast.literal_eval(args)
             return re.sub(old, new, value)
 
+        @staticmethod
+        def convert_replace_str_tuple(value: Any, args: str) -> str:
+            """
+            Apply multiple replacements on a string.
+            args should be a string representing a list/tuple of (old, new) pairs.
+            Example: '(("old1", "new1"), ("old2", "new2"))'
+            """
+            if isinstance(value, dict):
+                value = MetadataFormatter.convert_to_geojson(value)
+            elif not isinstance(value, str):
+                raise TypeError(
+                    f"convert_replace_str_tuple expects a string or a dict (apply to_geojson). "
+                    f"Got {type(value)}: {value}"
+                )
+
+            # args sera une chaîne représentant une liste/tuple de tuples
+            replacements = ast.literal_eval(args)
+
+            if not isinstance(replacements, (list, tuple)):
+                raise TypeError(
+                    f"convert_replace_str_tuple expects a list/tuple of (old,new) pairs. "
+                    f"Got {type(replacements)}: {replacements}"
+                )
+
+            for old, new in replacements:
+                value = re.sub(old, new, value)
+
+            return value
+
         @staticmethod
         def convert_ceda_collection_name(value: str) -> str:
             data_regex = re.compile(r"/data/(?P<name>.+?)/?$")
@@ -580,6 +612,45 @@ def format_metadata(search_param: str, *args: Any, **kwargs: Any) -> str:
                 result[key] = match.value
             return result
 
+        @staticmethod
+        def convert_dict_filter_and_sub(
+            input_dict: dict[Any, Any], args: str
+        ) -> Union[dict[Any, Any], list[Any]]:
+            """Fitlers dict items using jsonpath and then apply recursive_sub_str"""
+            jsonpath_filter_str, old, new = ast.literal_eval(args)
+            filtered = MetadataFormatter.convert_dict_filter(
+                input_dict, jsonpath_filter_str
+            )
+            args_str = f"('{old}', '{new}')"
+            return MetadataFormatter.convert_recursive_sub_str(filtered, args_str)
+
+        @staticmethod
+        def convert_from_alternate(
+            input_obj: dict[str, Any], value: str
+        ) -> dict[str, Any]:
+            """
+            Update assets using given alternate.
+            """
+            result: dict[str, Any] = {}
+            for k, v in input_obj.items():
+                if not isinstance(v, dict):
+                    continue
+
+                alt_dict = deepcopy(v).get("alternate")
+                if not isinstance(alt_dict, dict):
+                    continue
+
+                value_entry = alt_dict.pop(value, None)
+                if not isinstance(value_entry, dict):
+                    continue
+
+                result[k] = v | value_entry | {"alternate": alt_dict}
+
+                if len(result[k]["alternate"]) == 0:
+                    del result[k]["alternate"]
+
+            return result
+
         @staticmethod
         def convert_slice_str(string: str, args: str) -> str:
             cmin, cmax, cstep = [
@@ -591,6 +662,8 @@ def format_metadata(search_param: str, *args: Any, **kwargs: Any) -> str:
         @staticmethod
         def convert_to_lower(string: str) -> str:
             """Convert a string to lowercase."""
+            if string == NOT_AVAILABLE:
+                return string
             return string.lower()
 
         @staticmethod
@@ -598,6 +671,13 @@ def format_metadata(search_param: str, *args: Any, **kwargs: Any) -> str:
             """Convert a string to uppercase."""
             return string.upper()
 
+        @staticmethod
+        def convert_to_title(string: str) -> str:
+            """Convert a string to title case."""
+            if string == NOT_AVAILABLE:
+                return string
+            return string.title()
+
         @staticmethod
         def convert_fake_l2a_title_from_l1c(string: str) -> str:
             id_regex = re.compile(
@@ -1362,17 +1442,30 @@ def format_query_params(
         error_context,
     )
 
-    for eodag_search_key, provider_search_key in queryables.items():
+    for eodag_search_key, provider_search_param in queryables.items():
         user_input = query_dict[eodag_search_key]
 
-        if COMPLEX_QS_REGEX.match(provider_search_key):
-            parts = provider_search_key.split("=")
+        if provider_search_param == user_input:
+            # means the mapping is to be passed as is, in which case we
+            # readily register it
+            if (
+                eodag_search_key in query_params
+                and isinstance(query_params[eodag_search_key], dict)
+                and isinstance(user_input, dict)
+            ):
+                query_params[eodag_search_key].update(user_input)
+            else:
+                query_params[eodag_search_key] = user_input
+            continue
+
+        if COMPLEX_QS_REGEX.match(provider_search_param):
+            parts = provider_search_param.split("=")
             if len(parts) == 1:
                 formatted_query_param = format_metadata(
-                    provider_search_key, product_type, **query_dict
+                    provider_search_param, product_type, **query_dict
                 )
                 formatted_query_param = formatted_query_param.replace("'", '"')
-                if "{{" in provider_search_key:
+                if "{{" in provider_search_param:
                     # retrieve values from hashes where keys are given in the param
                     if "}[" in formatted_query_param:
                         formatted_query_param = _resolve_hashes(formatted_query_param)
@@ -1396,7 +1489,7 @@ def format_query_params(
                     provider_value, product_type, **query_dict
                 )
         else:
-            query_params[provider_search_key] = user_input
+            query_params[provider_search_param] = user_input
     # Now get all the literal search params (i.e params to be passed "as is"
     # in the search request)
     # ignore additional_params if it isn't a dictionary
@@ -1527,7 +1620,15 @@ def _get_queryables(
                     config.discover_metadata.get("metadata_pattern", "")
                 )
                 search_param_cfg = config.discover_metadata.get("search_param", "")
-                if pattern.match(eodag_search_key) and isinstance(
+                search_param_unparsed_cfg = config.discover_metadata.get(
+                    "search_param_unparsed", []
+                )
+                if (
+                    search_param_unparsed_cfg
+                    and eodag_search_key in search_param_unparsed_cfg
+                ):
+                    queryables[eodag_search_key] = user_input
+                elif pattern.match(eodag_search_key) and isinstance(
                     search_param_cfg, str
                 ):
                     search_param = search_param_cfg.format(metadata=eodag_search_key)

eodag/cli.py

@@ -42,13 +42,14 @@ Commands:
 
 from __future__ import annotations
 
+import functools
 import json
 import os
 import shutil
 import sys
 import textwrap
 from importlib.metadata import metadata
-from typing import TYPE_CHECKING, Any, Mapping
+from typing import TYPE_CHECKING, Any, Callable, Mapping, Optional
 from urllib.parse import parse_qs
 
 import click
@@ -118,6 +119,22 @@ class MutuallyExclusiveOption(click.Option):
         return super(MutuallyExclusiveOption, self).handle_parse_result(ctx, opts, args)
 
 
+def _deprecated_cli(message: str, version: Optional[str] = None) -> Callable[..., Any]:
+    """Decorator to mark a CLI command as deprecated and print a bold yellow warning."""
+    version_msg = f" -- Deprecated since v{version}" if version else ""
+
+    def decorator(func: Callable[..., Any]) -> Callable[..., Any]:
+        @functools.wraps(func)
+        def wrapper(*args: Any, **kwargs: Any) -> Any:
+            full_message = f"DEPRECATED: {message}{version_msg}"
+            click.echo(click.style(full_message, fg="yellow", bold=True), err=True)
+            return func(*args, **kwargs)
+
+        return wrapper
+
+    return decorator
+
+
 @click.group(chain=True)
 @click.option(
     "-v",
@@ -631,9 +648,17 @@ def download(ctx: Context, **kwargs: Any) -> None:
 
 
 @eodag.command(
-    help="Start eodag HTTP server\n\n"
-    "Set EODAG_CORS_ALLOWED_ORIGINS environment variable to configure Cross-Origin Resource Sharing allowed origins as "
-    "comma-separated URLs (e.g. 'http://somewhere,htttp://somewhere.else')."
+    help="(deprecated) Start eodag HTTP server\n\n"
+    + (
+        click.style(
+            "Running a web server from the CLI is deprecated and will be removed in a future version.\n"
+            "This feature has been moved to its own repository: https://github.com/CS-SI/stac-fastapi-eodag\n\n",
+            fg="yellow",
+            bold=True,
+        )
+        + "Set EODAG_CORS_ALLOWED_ORIGINS environment variable to configure Cross-Origin Resource Sharing allowed "
+        "origins as comma-separated URLs (e.g. 'http://somewhere,http://somewhere.else')."
+    )
 )
 @click.option(
     "-f",
@@ -676,6 +701,13 @@ def download(ctx: Context, **kwargs: Any) -> None:
     help="Run in debug mode (for development purpose)",
 )
 @click.pass_context
+@_deprecated_cli(
+    message=(
+        "Running a web server from the CLI is deprecated and will be removed in a future version. "
+        "This feature has been moved to its own repository: https://github.com/CS-SI/stac-fastapi-eodag"
+    ),
+    version="3.9.0",
+)
 def serve_rest(
     ctx: Context,
     daemon: bool,

eodag/config.py

@@ -272,6 +272,8 @@ class PluginConfig(yaml.YAMLObject):
         search_param: str | dict[str, Any]
         #: Path to the metadata in search result
         metadata_path: str
+        #: list search parameters to send as is to the provider
+        search_param_unparsed: list[str]
         #: Whether an error must be raised when using a search parameter which is not queryable or not
         raise_mtd_discovery_error: bool
 
@@ -543,8 +545,6 @@ class PluginConfig(yaml.YAMLObject):
     #: :class:`~eodag.plugins.download.s3rest.S3RestDownload`
     #: At which level of the path part of the url the bucket can be found
     bucket_path_level: int
-    #: :class:`~eodag.plugins.download.aws.AwsDownload` Whether download is done from a requester-pays bucket or not
-    requester_pays: bool
     #: :class:`~eodag.plugins.download.aws.AwsDownload` S3 endpoint
     s3_endpoint: str
 
@@ -571,6 +571,9 @@ class PluginConfig(yaml.YAMLObject):
     #: :class:`~eodag.plugins.authentication.base.Authentication` Part of the search or download plugin configuration
     #: that needs authentication
     matching_conf: dict[str, Any]
+    #: :class:`~eodag.plugins.authentication.aws_auth.AwsAuth`
+    #: Whether download is done from a requester-pays bucket or not
+    requester_pays: bool
     #: :class:`~eodag.plugins.authentication.openid_connect.OIDCRefreshTokenBase`
     #: How the token should be used in the request
     token_provision: str

eodag/plugins/apis/ecmwf.py

@@ -46,6 +46,7 @@ from eodag.utils.logging import get_logging_verbose
 if TYPE_CHECKING:
     from typing import Any, Optional, Union
 
+    from mypy_boto3_s3 import S3ServiceResource
     from requests.auth import AuthBase
 
     from eodag.api.product import EOProduct
@@ -55,6 +56,7 @@ if TYPE_CHECKING:
     from eodag.types.download_args import DownloadConf
     from eodag.utils import DownloadedCallback, ProgressCallback, Unpack
 
+
 logger = logging.getLogger("eodag.apis.ecmwf")
 
 ECMWF_MARS_KNOWN_FORMATS = {"grib": "grib", "netcdf": "nc"}
@@ -171,7 +173,7 @@ class EcmwfApi(Api, ECMWFSearch):
     def download(
         self,
         product: EOProduct,
-        auth: Optional[Union[AuthBase, S3SessionKwargs]] = None,
+        auth: Optional[Union[AuthBase, S3SessionKwargs, S3ServiceResource]] = None,
         progress_callback: Optional[ProgressCallback] = None,
         wait: float = DEFAULT_DOWNLOAD_WAIT,
         timeout: float = DEFAULT_DOWNLOAD_TIMEOUT,

eodag/plugins/apis/usgs.py

@@ -57,6 +57,7 @@ from eodag.utils.exceptions import (
 )
 
 if TYPE_CHECKING:
+    from mypy_boto3_s3 import S3ServiceResource
     from requests.auth import AuthBase
 
     from eodag.api.search_result import SearchResult
@@ -296,7 +297,7 @@ class UsgsApi(Api):
     def download(
         self,
         product: EOProduct,
-        auth: Optional[Union[AuthBase, S3SessionKwargs]] = None,
+        auth: Optional[Union[AuthBase, S3SessionKwargs, S3ServiceResource]] = None,
         progress_callback: Optional[ProgressCallback] = None,
         wait: float = DEFAULT_DOWNLOAD_WAIT,
         timeout: float = DEFAULT_DOWNLOAD_TIMEOUT,

eodag/plugins/authentication/aws_auth.py

@@ -17,29 +17,72 @@
 # limitations under the License.
 from __future__ import annotations
 
-from typing import TYPE_CHECKING, Optional, cast
+import logging
+from typing import TYPE_CHECKING, Any, Optional, cast
+
+import boto3
+from botocore.exceptions import ClientError, ProfileNotFound
+from botocore.handlers import disable_signing
 
 from eodag.plugins.authentication.base import Authentication
 from eodag.types import S3SessionKwargs
+from eodag.utils.exceptions import AuthenticationError
 
 if TYPE_CHECKING:
-    from mypy_boto3_s3.client import S3Client
+    from mypy_boto3_s3 import S3Client, S3ServiceResource
+    from mypy_boto3_s3.service_resource import BucketObjectsCollection
 
     from eodag.config import PluginConfig
 
 
+logger = logging.getLogger("eodag.download.aws_auth")
+
+AWS_AUTH_ERROR_MESSAGES = [
+    "AccessDenied",
+    "InvalidAccessKeyId",
+    "SignatureDoesNotMatch",
+    "InvalidRequest",
+]
+
+
+def raise_if_auth_error(exception: ClientError, provider: str) -> None:
+    """Raises an error if given exception is an authentication error"""
+    err = cast(dict[str, str], exception.response["Error"])
+    if err["Code"] in AWS_AUTH_ERROR_MESSAGES and "key" in err["Message"].lower():
+        raise AuthenticationError(
+            f"Please check your credentials for {provider}.",
+            f"HTTP Error {exception.response['ResponseMetadata']['HTTPStatusCode']} returned.",
+            err["Code"] + ": " + err["Message"],
+        )
+
+
+def create_s3_session(**kwargs) -> boto3.Session:
+    """create s3 session based on available credentials
+
+    :param kwargs: keyword arguments containing credentials
+    :returns: boto3 Session
+    """
+    try:
+        s3_session = boto3.Session(**kwargs)
+    except ProfileNotFound:
+        raise AuthenticationError(
+            f"AWS profile {kwargs['profile_name']} not found, please check your credentials configuration"
+        )
+    return s3_session
+
+
 class AwsAuth(Authentication):
     """AWS authentication plugin
 
-    Authentication will use the first valid method within the following ones depending on which
-    parameters are available in the configuration:
+    The authentication method will be chosen depending on which parameters are available in the configuration:
 
-    * auth anonymously using no-sign-request
-    * auth using ``aws_profile``
-    * auth using ``aws_access_key_id`` and ``aws_secret_access_key``
-      (optionally ``aws_session_token``)
-    * auth using current environment (AWS environment variables and/or ``~/aws/*``),
-      will be skipped if AWS credentials are filled in eodag conf
+    * auth using ``profile_name`` (if credentials are given and contain ``aws_profile``)
+    * auth using ``aws_access_key_id``, ``aws_secret_access_key`` and optionally ``aws_session_token``
+      (if credentials are given but no ``aws_profile``)
+    * auth using current environment - AWS environment variables and/or ``~/.aws/*``
+      (if no credentials are given in config)
+    * auth anonymously using no-sign-request if no credentials are given in config and
+      auth using current environment failed
 
     :param provider: provider name
     :param config: Authentication plugin configuration:
@@ -47,41 +90,189 @@ class AwsAuth(Authentication):
         * :attr:`~eodag.config.PluginConfig.type` (``str``) (**mandatory**): AwsAuth
         * :attr:`~eodag.config.PluginConfig.auth_error_code` (``int``) (mandatory for ``creodias_s3``):
           which error code is returned in case of an authentication error
+        * :attr:`~eodag.config.PluginConfig.s3_endpoint` (``str``): s3 endpoint url
+        * :attr:`~eodag.config.PluginConfig.requester_pays` (``bool``): whether download is done
+          from a requester-pays bucket or not; default: ``False``
 
     """
 
-    s3_client: S3Client
-
     def __init__(self, provider: str, config: PluginConfig) -> None:
         super(AwsAuth, self).__init__(provider, config)
-        self.aws_access_key_id: Optional[str] = None
-        self.aws_secret_access_key: Optional[str] = None
-        self.aws_session_token: Optional[str] = None
-        self.profile_name: Optional[str] = None
+        self.s3_session: Optional[boto3.Session] = None
+        self.s3_resource: Optional[S3ServiceResource] = None
+        # set default for requester_pays if not given
+        self.config.__dict__.setdefault("requester_pays", False)
+
+    def _create_s3_session_from_credentials(self) -> boto3.Session:
+        credentials = getattr(self.config, "credentials", {}) or {}
+        if "aws_profile" in credentials:
+            return create_s3_session(profile_name=credentials["aws_profile"])
+        # auth using aws keys
+        elif credentials:
+            s3_session_kwargs: S3SessionKwargs = {
+                "aws_access_key_id": credentials["aws_access_key_id"],
+                "aws_secret_access_key": credentials["aws_secret_access_key"],
+            }
+            if credentials.get("aws_session_token"):
+                s3_session_kwargs["aws_session_token"] = credentials[
+                    "aws_session_token"
+                ]
+            return create_s3_session(**s3_session_kwargs)
+        else:
+            # auth using env variables or ~/.aws
+            return create_s3_session()
+
+    def _create_s3_resource(self) -> S3ServiceResource:
+        """create s3 resource based on s3 session"""
+        if not self.s3_session:
+            self.s3_session = self._create_s3_session_from_credentials()
+        endpoint_url = getattr(self.config, "s3_endpoint", None)
+        if self.s3_session.get_credentials():
+            return self.s3_session.resource(
+                service_name="s3",
+                endpoint_url=endpoint_url,
+            )
+        # could not auth using credentials: use no-sign-request strategy
+        s3_resource = boto3.resource(service_name="s3", endpoint_url=endpoint_url)
+        s3_resource.meta.client.meta.events.register(
+            "choose-signer.s3.*", disable_signing
+        )
+        return s3_resource
+
+    def get_s3_client(self) -> S3Client:
+        """Get S3 client from S3 resource
 
-    def authenticate(self) -> S3SessionKwargs:
+        :returns: boto3 client
+        """
+        if not self.s3_resource:
+            self.s3_resource = self._create_s3_resource()
+        return self.s3_resource.meta.client
+
+    def authenticate(self) -> S3ServiceResource:
         """Authenticate
 
-        :returns: dict containing AWS/boto3 non-empty credentials
+        :returns: S3 Resource created based on an S3 session
         """
-        credentials = getattr(self.config, "credentials", {}) or {}
-        self.aws_access_key_id = credentials.get(
-            "aws_access_key_id", self.aws_access_key_id
-        )
-        self.aws_secret_access_key = credentials.get(
-            "aws_secret_access_key", self.aws_secret_access_key
-        )
-        self.aws_session_token = credentials.get(
-            "aws_session_token", self.aws_session_token
+        self.s3_resource = self._create_s3_resource()
+        return self.s3_resource
+
+    def _get_authenticated_objects(
+        self, bucket_name: str, prefix: str
+    ) -> BucketObjectsCollection:
+        """Get boto3 authenticated objects for the given bucket
+
+        :param bucket_name: Bucket containg objects
+        :param prefix: Prefix used to filter objects
+        :returns: The boto3 authenticated objects
+        """
+        if not self.s3_resource:
+            self.s3_resource = self._create_s3_resource()
+        try:
+            if self.config.requester_pays:
+                objects = self.s3_resource.Bucket(bucket_name).objects.filter(
+                    RequestPayer="requester"
+                )
+            else:
+                objects = self.s3_resource.Bucket(bucket_name).objects
+            list(objects.filter(Prefix=prefix).limit(1))
+            if objects:
+                logger.debug(
+                    "Authentication for bucket %s succeeded; returning available objects",
+                    bucket_name,
+                )
+                return objects
+        except ClientError as e:
+            if e.response.get("Error", {}).get("Code", {}) in AWS_AUTH_ERROR_MESSAGES:
+                pass
+            else:
+                raise e
+        logger.debug(
+            "Authentication for bucket %s failed, please check the credentials",
+            bucket_name,
         )
-        self.profile_name = credentials.get("aws_profile", self.profile_name)
-
-        auth_dict = cast(
-            S3SessionKwargs,
-            {
-                k: getattr(self, k)
-                for k in S3SessionKwargs.__annotations__
-                if getattr(self, k, None)
-            },
+
+        raise AuthenticationError(
+            "Unable do authenticate on s3://%s using credendials configuration"
+            % bucket_name
         )
-        return auth_dict
+
+    def authenticate_objects(
+        self,
+        bucket_names_and_prefixes: list[tuple[str, Optional[str]]],
+    ) -> dict[str, BucketObjectsCollection]:
+        """
+        Authenticates with s3 and retrieves the available objects
+
+        :param bucket_names_and_prefixes: list of bucket names and corresponding path prefixes
+        :raises AuthenticationError: authentication is not possible
+        :return: authenticated objects per bucket
+        """
+
+        authenticated_objects: dict[str, Any] = {}
+        auth_error_messages: set[str] = set()
+        for _, pack in enumerate(bucket_names_and_prefixes):
+
+            bucket_name, prefix = pack
+            if not prefix:
+                continue
+            if bucket_name not in authenticated_objects:
+                # get Prefixes longest common base path
+                common_prefix = ""
+                prefix_split = prefix.split("/")
+                prefixes_in_bucket = len(
+                    [p for b, p in bucket_names_and_prefixes if b == bucket_name]
+                )
+                for i in range(1, len(prefix_split)):
+                    common_prefix = "/".join(prefix_split[0:i])
+                    if (
+                        len(
+                            [
+                                p
+                                for b, p in bucket_names_and_prefixes
+                                if p and b == bucket_name and common_prefix in p
+                            ]
+                        )
+                        < prefixes_in_bucket
+                    ):
+                        common_prefix = "/".join(prefix_split[0 : i - 1])
+                        break
+                try:
+                    # connect to aws s3 and get bucket auhenticated objects
+                    authenticated_objects[
+                        bucket_name
+                    ] = self._get_authenticated_objects(bucket_name, common_prefix)
+
+                except AuthenticationError as e:
+                    logger.warning("Unexpected error: %s" % e)
+                    logger.warning("Skipping %s/%s" % (bucket_name, prefix))
+                    auth_error_messages.add(str(e))
+                except ClientError as e:
+                    raise_if_auth_error(e, self.provider)
+                    logger.warning("Unexpected error: %s" % e)
+                    logger.warning("Skipping %s/%s" % (bucket_name, prefix))
+                    auth_error_messages.add(str(e))
+
+        # could not auth on any bucket
+        if not authenticated_objects:
+            raise AuthenticationError(", ".join(auth_error_messages))
+        return authenticated_objects
+
+    def get_rio_env(self) -> dict[str, Any]:
+        """Get rasterio environment variables needed for data access authentication.
+
+        :returns: The rasterio environement variables
+        """
+        rio_env_kwargs = {}
+        if endpoint_url := getattr(self.config, "s3_endpoint", None):
+            rio_env_kwargs["endpoint_url"] = endpoint_url.split("://")[-1]
+
+        if self.s3_session is None:
+            self.authenticate()
+
+        if self.config.requester_pays:
+            rio_env_kwargs["requester_pays"] = True
+
+        return {
+            "session": self.s3_session,
+            **rio_env_kwargs,
+        }