eodag 3.0.0b2__py3-none-any.whl → 3.0.1__py3-none-any.whl

eodag/plugins/download/aws.py

@@ -34,6 +34,7 @@ from typing import (
     Optional,
     Set,
     Tuple,
+    TypedDict,
     Union,
     cast,
 )
@@ -69,6 +70,7 @@ from eodag.utils.exceptions import (
     AuthenticationError,
     DownloadError,
     MisconfiguredError,
+    NoMatchingProductType,
     NotAvailableError,
     TimeOutError,
 )
@@ -213,23 +215,36 @@ class AwsDownload(Download):
     """Download on AWS using S3 protocol.
 
     :param provider: provider name
-    :type provider: str
     :param config: Download plugin configuration:
 
-        * ``config.base_uri`` (str) - s3 endpoint url
-        * ``config.requester_pays`` (bool) - (optional) whether download is done from a
-          requester-pays bucket or not
-        * ``config.flatten_top_dirs`` (bool) - (optional) flatten directory structure
-        * ``config.products`` (dict) - (optional) product_type specific configuration
-        * ``config.ignore_assets`` (bool) - (optional) ignore assets and download using downloadLink
+        * :attr:`~eodag.config.PluginConfig.type` (``str``) (**mandatory**): AwsDownload
+        * :attr:`~eodag.config.PluginConfig.base_uri` (``str``) (**mandatory**): s3 endpoint url
+        * :attr:`~eodag.config.PluginConfig.requester_pays` (``bool``): whether download is done
+          from a requester-pays bucket or not; default: ``False``
+        * :attr:`~eodag.config.PluginConfig.flatten_top_dirs` (``bool``): if the directory structure
+          should be flattened; default: ``True``
+        * :attr:`~eodag.config.PluginConfig.ignore_assets` (``bool``): ignore assets and download
+          using ``downloadLink``; default: ``False``
+        * :attr:`~eodag.config.PluginConfig.ssl_verify` (``bool``): if the ssl certificates should
+          be verified in requests; default: ``True``
+        * :attr:`~eodag.config.PluginConfig.bucket_path_level` (``int``): at which level of the
+          path part of the url the bucket can be found; If no bucket_path_level is given, the bucket
+          is taken from the first element of the netloc part.
+        * :attr:`~eodag.config.PluginConfig.products` (``Dict[str, Dict[str, Any]``): product type
+          specific config; the keys are the product types, the values are dictionaries which can contain the keys:
+
+          * **default_bucket** (``str``): bucket where the product type can be found
+          * **complementary_url_key** (``str``): keys to add additional urls
+          * **build_safe** (``bool``): if a SAFE (Standard Archive Format for Europe) product should
+            be created; used for Sentinel products; default: False
+          * **fetch_metadata** (``Dict[str, Any]``): config for metadata to be fetched for the SAFE product
 
-    :type config: :class:`~eodag.config.PluginConfig`
     """
 
     def __init__(self, provider: str, config: PluginConfig) -> None:
         super(AwsDownload, self).__init__(provider, config)
         self.requester_pays = getattr(self.config, "requester_pays", False)
-        self.s3_session = None
+        self.s3_session: Optional[boto3.session.Session] = None
 
     def download(
         self,
@@ -246,25 +261,20 @@ class AwsDownload(Download):
         SAFE-build is configured for a given provider and product type.
         If the product title is configured to be updated during download and
         SAFE-formatted, its destination path will be:
-        `{outputs_prefix}/{title}`
+        `{output_dir}/{title}`
 
         :param product: The EO product to download
-        :type product: :class:`~eodag.api.product._product.EOProduct`
         :param auth: (optional) authenticated object
-        :type auth: Union[AuthBase, Dict[str, str]]
         :param progress_callback: (optional) A method or a callable object
                                   which takes a current size and a maximum
                                   size as inputs and handle progress bar
                                   creation and update to give the user a
                                   feedback on the download progress
-        :type progress_callback: :class:`~eodag.utils.ProgressCallback` or None
-        :param kwargs: `outputs_prefix` (str), `extract` (bool), `delete_archive` (bool)
+        :param kwargs: `output_dir` (str), `extract` (bool), `delete_archive` (bool)
                         and `dl_url_params` (dict) can be provided as additional kwargs
                         and will override any other values defined in a configuration
                         file or with environment variables.
-        :type kwargs: Union[str, bool, dict]
         :returns: The absolute path to the downloaded product in the local filesystem
-        :rtype: str
         """
         if auth is None:
             auth = {}
@@ -404,13 +414,9 @@ class AwsDownload(Download):
         - get file path
         - create directories
         :param product: product to be downloaded
-        :type product: EOProduct
         :param progress_callback: progress callback to be used
-        :type progress_callback: ProgressCallback
         :param kwargs: additional arguments
-        :type kwargs: Any
         :return: local path and file name
-        :rtype: Tuple[str, Optional[str]]
         """
         product_local_path, record_filename = self._prepare_download(
             product, progress_callback=progress_callback, **kwargs
@@ -432,9 +438,7 @@ class AwsDownload(Download):
         """
         updates the product properties with fetch metadata if safe build is enabled
         :param build_safe: if safe build is enabled
-        :type build_safe: bool
         :param product: product to be updated
-        :type product: EOProduct
         """
         product_conf = getattr(self.config, "products", {}).get(
             product.product_type, {}
@@ -480,13 +484,9 @@ class AwsDownload(Download):
         """
         retrieves the bucket names and path prefixes for the assets
         :param product: product for which the assets shall be downloaded
-        :type product: EOProduct
         :param asset_filter: text for which the assets should be filtered
-        :type asset_filter: str
         :param ignore_assets: if product instead of individual assets should be used
-        :type ignore_assets: bool
         :return: tuples of bucket names and prefixes
-        :rtype: List[Tuple[str, Optional[str]]]
         """
         # if assets are defined, use them instead of scanning product.location
         if len(product.assets) > 0 and not ignore_assets:
@@ -528,11 +528,8 @@ class AwsDownload(Download):
         authenticates with s3 and retrieves the available objects
         raises an error when authentication is not possible
         :param bucket_names_and_prefixes: list of bucket names and corresponding path prefixes
-        :type bucket_names_and_prefixes: List[Tuple[str, Optional[str]]]
         :param auth: authentication information
-        :type auth: Dict[str, str]
         :return: authenticated objects per bucket, list of available objects
-        :rtype: Tuple[Dict[str, Any], ResourceCollection[Any]]
         """
         if not isinstance(auth, (dict, type(None))):
             raise AuthenticationError(
@@ -602,17 +599,11 @@ class AwsDownload(Download):
         """
         retrieve unique product chunks based on authenticated objects and asset filters
         :param bucket_names_and_prefixes: list of bucket names and corresponding path prefixes
-        :type bucket_names_and_prefixes: List[Tuple[str, Optional[str]]]
         :param authenticated_objects: available objects per bucket
-        :type authenticated_objects: Dict[str, Any]
         :param asset_filter: text for which assets should be filtered
-        :type asset_filter: str
         :param ignore_assets: if product instead of individual assets should be used
-        :type ignore_assets: bool
         :param product: product that shall be downloaded
-        :type product: EOProduct
         :return: set of product chunks that can be downloaded
-        :rtype: Set[Any]
         """
         product_chunks: List[Any] = []
         for bucket_name, prefix in bucket_names_and_prefixes:
@@ -637,19 +628,20 @@ class AwsDownload(Download):
                 raise NotAvailableError(
                     rf"No file basename matching re.fullmatch(r'{asset_filter}') was found in {product.remote_location}"
                 )
+
+        if not unique_product_chunks:
+            raise NoMatchingProductType("No product found to download.")
+
         return unique_product_chunks
 
     def _raise_if_auth_error(self, exception: ClientError) -> None:
         """Raises an error if given exception is an authentication error"""
-        err = exception.response["Error"]
+        err = cast(Dict[str, str], exception.response["Error"])
         if err["Code"] in AWS_AUTH_ERROR_MESSAGES and "key" in err["Message"].lower():
             raise AuthenticationError(
-                "HTTP error {} returned\n{}: {}\nPlease check your credentials for {}".format(
-                    exception.response["ResponseMetadata"]["HTTPStatusCode"],
-                    err["Code"],
-                    err["Message"],
-                    self.provider,
-                )
+                f"Please check your credentials for {self.provider}.",
+                f"HTTP Error {exception.response['ResponseMetadata']['HTTPStatusCode']} returned.",
+                err["Code"] + ": " + err["Message"],
             )
 
     def _stream_download_dict(
@@ -662,27 +654,20 @@ class AwsDownload(Download):
         **kwargs: Unpack[DownloadConf],
     ) -> StreamResponse:
         r"""
-        Returns dictionnary of :class:`~fastapi.responses.StreamingResponse` keyword-arguments.
+        Returns dictionary of :class:`~fastapi.responses.StreamingResponse` keyword-arguments.
         It contains a generator to streamed download chunks and the response headers.
 
         :param product: The EO product to download
-        :type product: :class:`~eodag.api.product._product.EOProduct`
         :param auth: (optional) The configuration of a plugin of type Authentication
-        :type auth: :class:`~eodag.config.PluginConfig`
         :param progress_callback: (optional) A progress callback
-        :type progress_callback: :class:`~eodag.utils.ProgressCallback`
         :param wait: (optional) If download fails, wait time in minutes between two download tries
-        :type wait: int
         :param timeout: (optional) If download fails, maximum time in minutes before stop retrying
                         to download
-        :type timeout: int
-        :param kwargs: `outputs_prefix` (str), `extract` (bool), `delete_archive` (bool)
+        :param kwargs: `output_dir` (str), `extract` (bool), `delete_archive` (bool)
                         and `dl_url_params` (dict) can be provided as additional kwargs
                         and will override any other values defined in a configuration
                         file or with environment variables.
-        :type kwargs: Union[str, bool, dict]
-        :returns: Dictionnary of :class:`~fastapi.responses.StreamingResponse` keyword-arguments
-        :rtype: dict
+        :returns: Dictionary of :class:`~fastapi.responses.StreamingResponse` keyword-arguments
         """
         if progress_callback is None:
             logger.info(
@@ -773,7 +758,7 @@ class AwsDownload(Download):
         build_safe: bool,
         progress_callback: ProgressCallback,
         assets_values: List[Dict[str, Any]],
-    ) -> Iterator[Tuple[str, datetime, int, Any, Iterator[Any]]]:
+    ) -> Iterator[Any]:
         """Yield product data chunks"""
 
         chunk_size = 4096 * 1024
@@ -784,7 +769,6 @@ class AwsDownload(Download):
             product_chunk: Any, progress_callback: ProgressCallback
         ) -> Any:
             try:
-
                 chunk_start = 0
                 chunk_end = chunk_start + chunk_size - 1
 
@@ -861,13 +845,9 @@ class AwsDownload(Download):
         """Get rasterio environment variables needed for data access authentication.
 
         :param bucket_name: Bucket containg objects
-        :type bucket_name: str
         :param prefix: Prefix used to try auth
-        :type prefix: str
-        :param auth_dict: Dictionnary containing authentication keys
-        :type auth_dict: dict
+        :param auth_dict: Dictionary containing authentication keys
         :returns: The rasterio environement variables
-        :rtype: dict
         """
         if self.s3_session is not None:
             if self.requester_pays:
@@ -892,14 +872,10 @@ class AwsDownload(Download):
         Also expose ``s3_session`` as class variable if available.
 
         :param bucket_name: Bucket containg objects
-        :type bucket_name: str
         :param prefix: Prefix used to filter objects on auth try
                        (not used to filter returned objects)
-        :type prefix: str
-        :param auth_dict: Dictionnary containing authentication keys
-        :type auth_dict: dict
+        :param auth_dict: Dictionary containing authentication keys
         :returns: The boto3 authenticated objects
-        :rtype: :class:`~boto3.resources.collection.s3.Bucket.objectsCollection`
         """
         auth_methods: List[
             Callable[[str, str, Dict[str, str]], Optional[ResourceCollection]]
@@ -941,15 +917,15 @@ class AwsDownload(Download):
     ) -> Optional[ResourceCollection]:
         """Auth strategy using no-sign-request"""
 
-        s3_resource = boto3.resource(  # type: ignore
-            service_name="s3", endpoint_url=getattr(self.config, "base_uri", None)
+        s3_resource = boto3.resource(
+            service_name="s3", endpoint_url=getattr(self.config, "s3_endpoint", None)
         )
-        s3_resource.meta.client.meta.events.register(  # type: ignore
+        s3_resource.meta.client.meta.events.register(
             "choose-signer.s3.*", disable_signing
         )
-        objects = s3_resource.Bucket(bucket_name).objects  # type: ignore
-        list(objects.filter(Prefix=prefix).limit(1))  # type: ignore
-        return objects  # type: ignore
+        objects = s3_resource.Bucket(bucket_name).objects
+        list(objects.filter(Prefix=prefix).limit(1))
+        return objects
 
     def _get_authenticated_objects_from_auth_profile(
         self, bucket_name: str, prefix: str, auth_dict: Dict[str, str]
@@ -957,20 +933,20 @@ class AwsDownload(Download):
         """Auth strategy using RequestPayer=requester and ``aws_profile`` from provided credentials"""
 
         if "profile_name" in auth_dict.keys():
-            s3_session = boto3.session.Session(profile_name=auth_dict["profile_name"])  # type: ignore
-            s3_resource = s3_session.resource(  # type: ignore
+            s3_session = boto3.session.Session(profile_name=auth_dict["profile_name"])
+            s3_resource = s3_session.resource(
                 service_name="s3",
-                endpoint_url=getattr(self.config, "base_uri", None),
+                endpoint_url=getattr(self.config, "s3_endpoint", None),
             )
             if self.requester_pays:
-                objects = s3_resource.Bucket(bucket_name).objects.filter(  # type: ignore
+                objects = s3_resource.Bucket(bucket_name).objects.filter(
                     RequestPayer="requester"
                 )
             else:
-                objects = s3_resource.Bucket(bucket_name).objects  # type: ignore
-            list(objects.filter(Prefix=prefix).limit(1))  # type: ignore
-            self.s3_session = s3_session  # type: ignore
-            return objects  # type: ignore
+                objects = s3_resource.Bucket(bucket_name).objects
+            list(objects.filter(Prefix=prefix).limit(1))
+            self.s3_session = s3_session
+            return objects
         else:
             return None
 
@@ -981,23 +957,35 @@ class AwsDownload(Download):
         from provided credentials"""
 
         if all(k in auth_dict for k in ("aws_access_key_id", "aws_secret_access_key")):
-            s3_session = boto3.session.Session(  # type: ignore
-                aws_access_key_id=auth_dict["aws_access_key_id"],
-                aws_secret_access_key=auth_dict["aws_secret_access_key"],
+            S3SessionKwargs = TypedDict(
+                "S3SessionKwargs",
+                {
+                    "aws_access_key_id": str,
+                    "aws_secret_access_key": str,
+                    "aws_session_token": str,
+                },
+                total=False,
             )
-            s3_resource = s3_session.resource(  # type: ignore
+            s3_session_kwargs: S3SessionKwargs = {
+                "aws_access_key_id": auth_dict["aws_access_key_id"],
+                "aws_secret_access_key": auth_dict["aws_secret_access_key"],
+            }
+            if auth_dict.get("aws_session_token"):
+                s3_session_kwargs["aws_session_token"] = auth_dict["aws_session_token"]
+            s3_session = boto3.session.Session(**s3_session_kwargs)
+            s3_resource = s3_session.resource(
                 service_name="s3",
-                endpoint_url=getattr(self.config, "base_uri", None),
+                endpoint_url=getattr(self.config, "s3_endpoint", None),
             )
             if self.requester_pays:
-                objects = s3_resource.Bucket(bucket_name).objects.filter(  # type: ignore
+                objects = s3_resource.Bucket(bucket_name).objects.filter(
                     RequestPayer="requester"
                 )
             else:
-                objects = s3_resource.Bucket(bucket_name).objects  # type: ignore
-            list(objects.filter(Prefix=prefix).limit(1))  # type: ignore
-            self.s3_session = s3_session  # type: ignore
-            return objects  # type: ignore
+                objects = s3_resource.Bucket(bucket_name).objects
+            list(objects.filter(Prefix=prefix).limit(1))
+            self.s3_session = s3_session
+            return objects
         else:
             return None
 
@@ -1006,19 +994,19 @@ class AwsDownload(Download):
     ) -> Optional[ResourceCollection]:
         """Auth strategy using RequestPayer=requester and current environment"""
 
-        s3_session = boto3.session.Session()  # type: ignore
-        s3_resource = s3_session.resource(  # type: ignore
-            service_name="s3", endpoint_url=getattr(self.config, "base_uri", None)
+        s3_session = boto3.session.Session()
+        s3_resource = s3_session.resource(
+            service_name="s3", endpoint_url=getattr(self.config, "s3_endpoint", None)
         )
         if self.requester_pays:
-            objects = s3_resource.Bucket(bucket_name).objects.filter(  # type: ignore
+            objects = s3_resource.Bucket(bucket_name).objects.filter(
                 RequestPayer="requester"
             )
         else:
-            objects = s3_resource.Bucket(bucket_name).objects  # type: ignore
-        list(objects.filter(Prefix=prefix).limit(1))  # type: ignore
-        self.s3_session = s3_session  # type: ignore
-        return objects  # type: ignore
+            objects = s3_resource.Bucket(bucket_name).objects
+        list(objects.filter(Prefix=prefix).limit(1))
+        self.s3_session = s3_session
+        return objects
 
     def get_product_bucket_name_and_prefix(
         self, product: EOProduct, url: Optional[str] = None
@@ -1026,11 +1014,8 @@ class AwsDownload(Download):
         """Extract bucket name and prefix from product URL
 
         :param product: The EO product to download
-        :type product: :class:`~eodag.api.product._product.EOProduct`
         :param url: (optional) URL to use as product.location
-        :type url: str
         :returns: bucket_name and prefix as str
-        :rtype: tuple
         """
         if url is None:
             url = product.location