eodag 3.0.1__py3-none-any.whl → 3.1.0__py3-none-any.whl

eodag/plugins/search/cop_marine.py

@@ -22,7 +22,7 @@ import logging
 import os
 import re
 from datetime import datetime
-from typing import TYPE_CHECKING, Any, Dict, List, Optional, Tuple, cast
+from typing import TYPE_CHECKING, Any, Optional, cast
 from urllib.parse import urlsplit
 
 import boto3
@@ -69,8 +69,8 @@ def _get_date_from_yyyymmdd(date_str: str, item_key: str) -> Optional[datetime]:
 
 
 def _get_dates_from_dataset_data(
-    dataset_item: Dict[str, Any]
-) -> Optional[Dict[str, str]]:
+    dataset_item: dict[str, Any]
+) -> Optional[dict[str, str]]:
     dates = {}
     if "start_datetime" in dataset_item["properties"]:
         dates["start"] = dataset_item["properties"]["start_datetime"]
@@ -96,7 +96,7 @@ def _get_s3_client(endpoint_url: str) -> S3Client:
     )
 
 
-def _check_int_values_properties(properties: Dict[str, Any]):
+def _check_int_values_properties(properties: dict[str, Any]):
     # remove int values with a bit length of more than 64 from the properties
     invalid = []
     for prop, prop_value in properties.items():
@@ -134,7 +134,7 @@ class CopMarineSearch(StaticStacSearch):
 
     def _get_product_type_info(
         self, product_type: str
-    ) -> Tuple[Dict[str, Any], List[Dict[str, Any]]]:
+    ) -> tuple[dict[str, Any], list[dict[str, Any]]]:
         """Fetch product type and associated datasets info"""
 
         fetch_url = cast(str, self.config.discover_product_types["fetch_url"]).format(
@@ -183,13 +183,23 @@ class CopMarineSearch(StaticStacSearch):
         product_id: str,
         s3_url: str,
         product_type: str,
-        dataset_item: Dict[str, Any],
-        collection_dict: Dict[str, Any],
+        dataset_item: dict[str, Any],
+        collection_dict: dict[str, Any],
     ):
+        # try to find date(s) in product id
+        item_dates = re.findall(r"(\d{4})(0[1-9]|1[0-2])([0-3]\d)", product_id)
+        if not item_dates:
+            item_dates = re.findall(r"_(\d{4})(0[1-9]|1[0-2])", product_id)
+        use_dataset_dates = not bool(item_dates)
         for obj in collection_objects["Contents"]:
             if product_id in obj["Key"]:
                 return self._create_product(
-                    product_type, obj["Key"], s3_url, dataset_item, collection_dict
+                    product_type,
+                    obj["Key"],
+                    s3_url,
+                    dataset_item,
+                    collection_dict,
+                    use_dataset_dates,
                 )
         return None
 
@@ -198,8 +208,8 @@ class CopMarineSearch(StaticStacSearch):
         product_type: str,
         item_key: str,
         s3_url: str,
-        dataset_item: Dict[str, Any],
-        collection_dict: Dict[str, Any],
+        dataset_item: dict[str, Any],
+        collection_dict: dict[str, Any],
         use_dataset_dates: bool = False,
     ) -> Optional[EOProduct]:
 
@@ -278,7 +288,7 @@ class CopMarineSearch(StaticStacSearch):
         self,
         prep: PreparedSearch = PreparedSearch(),
         **kwargs: Any,
-    ) -> Tuple[List[EOProduct], Optional[int]]:
+    ) -> tuple[list[EOProduct], Optional[int]]:
         """
         Implementation of search for the Copernicus Marine provider
         :param prep: object containing search parameterds
@@ -298,7 +308,7 @@ class CopMarineSearch(StaticStacSearch):
                 "parameter product type is required for search with cop_marine provider"
             )
         collection_dict, datasets_items_list = self._get_product_type_info(product_type)
-        products: List[EOProduct] = []
+        products: list[EOProduct] = []
         start_index = items_per_page * (page - 1) + 1
         num_total = 0
         for i, dataset_item in enumerate(datasets_items_list):

eodag/plugins/search/creodias_s3.py

@@ -17,31 +17,22 @@
 # limitations under the License.
 import logging
 from types import MethodType
-from typing import Any, List
+from typing import Any
 
-import boto3
-import botocore
 from botocore.exceptions import BotoCoreError
 
-from eodag.api.product import AssetsDict, EOProduct  # type: ignore
+from eodag.api.product import EOProduct  # type: ignore
 from eodag.api.search_result import RawSearchResult
-from eodag.config import PluginConfig
-from eodag.plugins.authentication.aws_auth import AwsAuth
 from eodag.plugins.search.qssearch import ODataV4Search
-from eodag.utils import guess_file_type
-from eodag.utils.exceptions import (
-    AuthenticationError,
-    MisconfiguredError,
-    NotAvailableError,
-    RequestError,
-)
+from eodag.utils.exceptions import RequestError
+from eodag.utils.s3 import update_assets_from_s3
 
-DATA_EXTENSIONS = ["jp2", "tiff", "nc", "grib"]
 logger = logging.getLogger("eodag.search.creodiass3")
 
 
 def patched_register_downloader(self, downloader, authenticator):
     """Add the download information to the product.
+
     :param self: product to which information should be added
     :param downloader: The download method that it can use
                       :class:`~eodag.plugins.download.base.Download` or
@@ -53,68 +44,13 @@ def patched_register_downloader(self, downloader, authenticator):
     self.register_downloader_only(downloader, authenticator)
     # and also update assets
     try:
-        _update_assets(self, downloader.config, authenticator)
+        update_assets_from_s3(
+            self, authenticator, getattr(downloader.config, "s3_endpoint", None)
+        )
     except BotoCoreError as e:
         raise RequestError.from_error(e, "could not update assets") from e
 
 
-def _update_assets(product: EOProduct, config: PluginConfig, auth: AwsAuth):
-    product.assets = {}
-    prefix = (
-        product.properties.get("productIdentifier", None).replace("/eodata/", "") + "/"
-    )
-    if prefix:
-        try:
-            auth_dict = auth.authenticate()
-            required_creds = ["aws_access_key_id", "aws_secret_access_key"]
-            if not all(x in auth_dict for x in required_creds):
-                raise MisconfiguredError(
-                    f"Incomplete credentials for {product.provider}, missing "
-                    f"{[x for x in required_creds if x not in auth_dict]}"
-                )
-            if not getattr(auth, "s3_client", None):
-                auth.s3_client = boto3.client(
-                    "s3",
-                    endpoint_url=config.s3_endpoint,
-                    aws_access_key_id=auth_dict["aws_access_key_id"],
-                    aws_secret_access_key=auth_dict["aws_secret_access_key"],
-                )
-            logger.debug("Listing assets in %s", prefix)
-            product.assets = AssetsDict(product)
-            for asset in auth.s3_client.list_objects(
-                Bucket=config.s3_bucket, Prefix=prefix, MaxKeys=300
-            )["Contents"]:
-                asset_basename = (
-                    asset["Key"].split("/")[-1] if "/" in asset["Key"] else asset["Key"]
-                )
-
-                if len(asset_basename) > 0 and asset_basename not in product.assets:
-                    role = (
-                        "data"
-                        if asset_basename.split(".")[-1] in DATA_EXTENSIONS
-                        else "metadata"
-                    )
-
-                    product.assets[asset_basename] = {
-                        "title": asset_basename,
-                        "roles": [role],
-                        "href": f"s3://{config.s3_bucket}/{asset['Key']}",
-                    }
-                    if mime_type := guess_file_type(asset["Key"]):
-                        product.assets[asset_basename]["type"] = mime_type
-            # update driver
-            product.driver = product.get_driver()
-
-        except botocore.exceptions.ClientError as e:
-            if str(auth.config.auth_error_code) in str(e):
-                raise AuthenticationError(
-                    f"Authentication failed on {config.base_uri} s3"
-                ) from e
-            raise NotAvailableError(
-                f"assets for product {prefix} could not be found"
-            ) from e
-
-
 class CreodiasS3Search(ODataV4Search):
     """
     ``CreodiasS3Search`` is an extension of :class:`~eodag.plugins.search.qssearch.ODataV4Search`,
@@ -133,7 +69,7 @@ class CreodiasS3Search(ODataV4Search):
 
     def normalize_results(
         self, results: RawSearchResult, **kwargs: Any
-    ) -> List[EOProduct]:
+    ) -> list[EOProduct]:
         """Build EOProducts from provider results"""
 
         products = super(CreodiasS3Search, self).normalize_results(results, **kwargs)

eodag/plugins/search/csw.py

@@ -19,7 +19,7 @@ from __future__ import annotations
 
 import logging
 import re
-from typing import TYPE_CHECKING, Any, Dict, List, Optional, Tuple, Union
+from typing import TYPE_CHECKING, Any, Optional, Union
 
 import pyproj
 from owslib.csw import CatalogueServiceWeb
@@ -60,13 +60,13 @@ class CSWSearch(Search):
         * :attr:`~eodag.config.PluginConfig.api_endpoint` (``str``) (**mandatory**): The endpoint of the
           provider's search interface
         * :attr:`~eodag.config.PluginConfig.version` (``str``): OGC Catalogue Service version; default: ``2.0.2``
-        * :attr:`~eodag.config.PluginConfig.search_definition` (``Dict[str, Any]``) (**mandatory**):
+        * :attr:`~eodag.config.PluginConfig.search_definition` (``dict[str, Any]``) (**mandatory**):
 
-          * **product_type_tags** (``List[Dict[str, Any]``): dict of product type tags
+          * **product_type_tags** (``list[dict[str, Any]``): dict of product type tags
           * **resource_location_filter** (``str``): regex string
-          * **date_tags** (``Dict[str, Any]``): tags for start and end
+          * **date_tags** (``dict[str, Any]``): tags for start and end
 
-        * :attr:`~eodag.config.PluginConfig.metadata_mapping` (``Dict[str, Any]``): The search plugins of this kind can
+        * :attr:`~eodag.config.PluginConfig.metadata_mapping` (``dict[str, Any]``): The search plugins of this kind can
           detect when a metadata mapping is "query-able", and get the semantics of how to format the query string
           parameter that enables to make a query on the corresponding metadata. To make a metadata query-able,
           just configure it in the metadata mapping to be a list of 2 items, the first one being the
@@ -107,7 +107,7 @@ class CSWSearch(Search):
         self,
         prep: PreparedSearch = PreparedSearch(),
         **kwargs: Any,
-    ) -> Tuple[List[EOProduct], Optional[int]]:
+    ) -> tuple[list[EOProduct], Optional[int]]:
         """Perform a search on a OGC/CSW-like interface"""
         product_type = kwargs.get("productType")
         if product_type is None:
@@ -117,7 +117,7 @@ class CSWSearch(Search):
             self.__init_catalog(**getattr(auth.config, "credentials", {}))
         else:
             self.__init_catalog()
-        results: List[EOProduct] = []
+        results: list[EOProduct] = []
         if self.catalog:
             provider_product_type = self.config.products[product_type]["productType"]
             for product_type_def in self.config.search_definition["product_type_tags"]:
@@ -229,12 +229,12 @@ class CSWSearch(Search):
 
     def __convert_query_params(
         self,
-        product_type_def: Dict[str, Any],
+        product_type_def: dict[str, Any],
         product_type: str,
-        params: Dict[str, Any],
-    ) -> Union[List[OgcExpression], List[List[OgcExpression]]]:
+        params: dict[str, Any],
+    ) -> Union[list[OgcExpression], list[list[OgcExpression]]]:
         """Translates eodag search to CSW constraints using owslib constraint classes"""
-        constraints: List[OgcExpression] = []
+        constraints: list[OgcExpression] = []
         # How the match should be performed (fuzzy, prefix, postfix or exact).
         # defaults to fuzzy
         pt_tag, matching = (

eodag/plugins/search/data_request_search.py

@@ -20,7 +20,7 @@ from __future__ import annotations
 import logging
 import time
 from datetime import datetime, timedelta, timezone
-from typing import TYPE_CHECKING, Any, Dict, List, Optional, Tuple, cast
+from typing import TYPE_CHECKING, Any, Optional, cast
 
 import requests
 
@@ -36,6 +36,7 @@ from eodag.utils import (
     DEFAULT_ITEMS_PER_PAGE,
     DEFAULT_MISSION_START_DATE,
     DEFAULT_PAGE,
+    DEFAULT_SEARCH_TIMEOUT,
     GENERIC_PRODUCT_TYPE,
     HTTP_REQ_TIMEOUT,
     USER_AGENT,
@@ -113,10 +114,10 @@ class DataRequestSearch(Search):
           * :attr:`~eodag.config.PluginConfig.DiscoverProductTypes.generic_product_type_id` (``str``): mapping for the
             product type id
           * :attr:`~eodag.config.PluginConfig.DiscoverProductTypes.generic_product_type_parsable_metadata`
-            (``Dict[str, str]``): mapping for product type metadata (e.g. ``abstract``, ``licence``) which can be parsed
+            (``dict[str, str]``): mapping for product type metadata (e.g. ``abstract``, ``licence``) which can be parsed
             from the provider result
           * :attr:`~eodag.config.PluginConfig.DiscoverProductTypes.generic_product_type_parsable_properties`
-            (``Dict[str, str]``): mapping for product type properties which can be parsed from the result that are not
+            (``dict[str, str]``): mapping for product type properties which can be parsed from the result and are not
             product type metadata
           * :attr:`~eodag.config.PluginConfig.DiscoverProductTypes.single_collection_fetch_url` (``str``): url to fetch
             data for a single collection; used if product type metadata is not available from the endpoint given in
@@ -125,7 +126,7 @@ class DataRequestSearch(Search):
             to be added to the :attr:`~eodag.config.PluginConfig.DiscoverProductTypes.fetch_url` to filter for a
             collection
           * :attr:`~eodag.config.PluginConfig.DiscoverProductTypes.single_product_type_parsable_metadata`
-            (``Dict[str, str]``): mapping for product type metadata returned by the endpoint given in
+            (``dict[str, str]``): mapping for product type metadata returned by the endpoint given in
             :attr:`~eodag.config.PluginConfig.DiscoverProductTypes.single_collection_fetch_url`.
 
         * :attr:`~eodag.config.PluginConfig.constraints_file_url` (``str``): url to fetch the constraints for a specific
@@ -133,7 +134,7 @@ class DataRequestSearch(Search):
         * :attr:`~eodag.config.PluginConfig.constraints_entry` (``str``): key in the json result where the constraints
           can be found; if not given, it is assumed that the constraints are on top level of the result, i.e.
           the result is an array of constraints
-        * :attr:`~eodag.config.PluginConfig.metadata_mapping` (``Dict[str, Any]``): The search plugins of this kind can
+        * :attr:`~eodag.config.PluginConfig.metadata_mapping` (``dict[str, Any]``): The search plugins of this kind can
           detect when a metadata mapping is "query-able", and get the semantics of how to format the query string
           parameter that enables to make a query on the corresponding metadata. To make a metadata query-able,
           just configure it in the metadata mapping to be a list of 2 items, the first one being the
@@ -186,7 +187,7 @@ class DataRequestSearch(Search):
                 )
                 if other_product_for_mapping:
                     other_product_type_def_params = self.get_product_type_def_params(
-                        other_product_for_mapping,  # **kwargs
+                        other_product_for_mapping,
                     )
                     product_type_metadata_mapping.update(
                         other_product_type_def_params.get("metadata_mapping", {})
@@ -207,10 +208,10 @@ class DataRequestSearch(Search):
             self.config.pagination["next_page_url_key_path"] = string_to_jsonpath(
                 self.config.pagination.get("next_page_url_key_path", None)
             )
-        self.download_info: Dict[str, Any] = {}
+        self.download_info: dict[str, Any] = {}
         self.data_request_id = None
 
-    def discover_product_types(self, **kwargs: Any) -> Optional[Dict[str, Any]]:
+    def discover_product_types(self, **kwargs: Any) -> Optional[dict[str, Any]]:
         """Fetch product types is disabled for `DataRequestSearch`
 
         :returns: empty dict
@@ -226,7 +227,7 @@ class DataRequestSearch(Search):
         self,
         prep: PreparedSearch = PreparedSearch(),
         **kwargs: Any,
-    ) -> Tuple[List[EOProduct], Optional[int]]:
+    ) -> tuple[list[EOProduct], Optional[int]]:
         """
         performs the search for a provider where several steps are required to fetch the data
         """
@@ -252,7 +253,7 @@ class DataRequestSearch(Search):
 
         # provider product type specific conf
         self.product_type_def_params = self.get_product_type_def_params(
-            product_type, **kwargs
+            product_type, format_variables=kwargs
         )
 
         # update config using provider product type definition metadata_mapping
@@ -262,7 +263,7 @@ class DataRequestSearch(Search):
         )
         if other_product_for_mapping:
             other_product_type_def_params = self.get_product_type_def_params(
-                other_product_for_mapping, **kwargs
+                other_product_for_mapping, format_variables=kwargs
             )
             self.config.metadata_mapping.update(
                 other_product_type_def_params.get("metadata_mapping", {})
@@ -308,7 +309,7 @@ class DataRequestSearch(Search):
             request_finished = True
 
         # loop to check search job status
-        search_timeout = int(getattr(self.config, "timeout", HTTP_REQ_TIMEOUT))
+        search_timeout = int(getattr(self.config, "timeout", DEFAULT_SEARCH_TIMEOUT))
         logger.info(
             f"checking status of request job {data_request_id} (timeout={search_timeout}s)"
         )
@@ -431,7 +432,7 @@ class DataRequestSearch(Search):
 
     def _get_result_data(
         self, data_request_id: str, items_per_page: int, page: int
-    ) -> Dict[str, Any]:
+    ) -> dict[str, Any]:
         page = page - 1 + self.config.pagination.get("start_page", 1)
         url = self.config.result_url.format(
             jobId=data_request_id, items_per_page=items_per_page, page=page
@@ -450,18 +451,18 @@ class DataRequestSearch(Search):
 
     def _convert_result_data(
         self,
-        result_data: Dict[str, Any],
+        result_data: dict[str, Any],
         data_request_id: str,
         product_type: str,
         **kwargs: Any,
-    ) -> Tuple[List[EOProduct], int]:
+    ) -> tuple[list[EOProduct], int]:
         """Build EOProducts from provider results"""
         results_entry = self.config.results_entry
         results = result_data[results_entry]
         logger.debug(
             "Adapting %s plugin results to eodag product representation" % len(results)
         )
-        products: List[EOProduct] = []
+        products: list[EOProduct] = []
         for result in results:
             product = EOProduct(
                 self.provider,
@@ -517,8 +518,8 @@ class DataRequestSearch(Search):
         return False
 
     def _apply_additional_filters(
-        self, result: Dict[str, Any], custom_filters: Dict[str, str]
-    ) -> Dict[str, Any]:
+        self, result: dict[str, Any], custom_filters: dict[str, str]
+    ) -> dict[str, Any]:
         filtered_result = []
         results_entry = self.config.results_entry
         results = result[results_entry]