eodag 3.0.0b3__py3-none-any.whl → 3.1.0__py3-none-any.whl

eodag/plugins/search/cop_marine.py

@@ -19,9 +19,10 @@ from __future__ import annotations
 
 import copy
 import logging
+import os
 import re
 from datetime import datetime
-from typing import TYPE_CHECKING, Any, Dict, List, Optional, Tuple, cast
+from typing import TYPE_CHECKING, Any, Optional, cast
 from urllib.parse import urlsplit
 
 import boto3
@@ -37,7 +38,7 @@ from eodag.config import PluginConfig
 from eodag.plugins.search import PreparedSearch
 from eodag.plugins.search.static_stac_search import StaticStacSearch
 from eodag.utils import get_bucket_name_and_prefix
-from eodag.utils.exceptions import UnsupportedProductType, ValidationError
+from eodag.utils.exceptions import RequestError, UnsupportedProductType, ValidationError
 
 if TYPE_CHECKING:
     from mypy_boto3_s3 import S3Client
@@ -67,6 +68,21 @@ def _get_date_from_yyyymmdd(date_str: str, item_key: str) -> Optional[datetime]:
         return date
 
 
+def _get_dates_from_dataset_data(
+    dataset_item: dict[str, Any]
+) -> Optional[dict[str, str]]:
+    dates = {}
+    if "start_datetime" in dataset_item["properties"]:
+        dates["start"] = dataset_item["properties"]["start_datetime"]
+        dates["end"] = dataset_item["properties"]["end_datetime"]
+    elif "datetime" in dataset_item["properties"]:
+        dates["start"] = dataset_item["properties"]["datetime"]
+        dates["end"] = dataset_item["properties"]["datetime"]
+    else:
+        return None
+    return dates
+
+
 def _get_s3_client(endpoint_url: str) -> S3Client:
     s3_session = boto3.Session()
     return s3_session.client(
@@ -80,7 +96,7 @@ def _get_s3_client(endpoint_url: str) -> S3Client:
     )
 
 
-def _check_int_values_properties(properties: Dict[str, Any]):
+def _check_int_values_properties(properties: dict[str, Any]):
     # remove int values with a bit length of more than 64 from the properties
     invalid = []
     for prop, prop_value in properties.items():
@@ -94,7 +110,21 @@ def _check_int_values_properties(properties: Dict[str, Any]):
 
 
 class CopMarineSearch(StaticStacSearch):
-    """class that implements search for the Copernicus Marine provider"""
+    """class that implements search for the Copernicus Marine provider
+
+    It calls :meth:`~eodag.plugins.search.static_stac_search.StaticStacSearch.discover_product_types`
+    inherited from :class:`~eodag.plugins.search.static_stac_search.StaticStacSearch`
+    but for the actual search a special method which fetches the urls of the available products from an S3 storage and
+    filters them has been written.
+
+    The configuration parameters are inherited from the parent and grand-parent classes. The
+    :attr:`~eodag.config.PluginConfig.DiscoverMetadata.auto_discovery` parameter in the
+    :attr:`~eodag.config.PluginConfig.discover_metadata` section has to be set to ``false`` and the
+    :attr:`~eodag.config.PluginConfig.DiscoverQueryables.fetch_url` in the
+    :attr:`~eodag.config.PluginConfig.discover_queryables` queryables section has to be set to ``null`` to
+    overwrite the default config from the stac provider configuration because those functionalities
+    are not available.
+    """
 
     def __init__(self, provider: str, config: PluginConfig):
         original_metadata_mapping = copy.deepcopy(config.metadata_mapping)
@@ -104,15 +134,13 @@ class CopMarineSearch(StaticStacSearch):
 
     def _get_product_type_info(
         self, product_type: str
-    ) -> Tuple[Dict[str, Any], List[Dict[str, Any]]]:
+    ) -> tuple[dict[str, Any], list[dict[str, Any]]]:
         """Fetch product type and associated datasets info"""
 
-        fetch_url = cast(
-            str,
-            self.config.discover_product_types["fetch_url"].format(
-                **self.config.__dict__
-            ),
+        fetch_url = cast(str, self.config.discover_product_types["fetch_url"]).format(
+            **self.config.__dict__
         )
+
         logger.debug("fetch data for collection %s", product_type)
         provider_product_type = self.config.products.get(product_type, {}).get(
             "productType", None
@@ -125,9 +153,14 @@ class CopMarineSearch(StaticStacSearch):
         )
         try:
             collection_data = requests.get(collection_url).json()
-        except requests.RequestException:
+        except requests.RequestException as exc:
+            if exc.errno == 404:
+                logger.error("product %s not found", product_type)
+                raise UnsupportedProductType(product_type)
             logger.error("data for product %s could not be fetched", product_type)
-            raise UnsupportedProductType(product_type)
+            raise RequestError.from_error(
+                exc, f"data for product {product_type} could not be fetched"
+            ) from exc
 
         datasets = []
         for link in [li for li in collection_data["links"] if li["rel"] == "item"]:
@@ -150,13 +183,23 @@ class CopMarineSearch(StaticStacSearch):
         product_id: str,
         s3_url: str,
         product_type: str,
-        dataset_item: Dict[str, Any],
-        collection_dict: Dict[str, Any],
+        dataset_item: dict[str, Any],
+        collection_dict: dict[str, Any],
     ):
+        # try to find date(s) in product id
+        item_dates = re.findall(r"(\d{4})(0[1-9]|1[0-2])([0-3]\d)", product_id)
+        if not item_dates:
+            item_dates = re.findall(r"_(\d{4})(0[1-9]|1[0-2])", product_id)
+        use_dataset_dates = not bool(item_dates)
         for obj in collection_objects["Contents"]:
             if product_id in obj["Key"]:
                 return self._create_product(
-                    product_type, obj["Key"], s3_url, dataset_item, collection_dict
+                    product_type,
+                    obj["Key"],
+                    s3_url,
+                    dataset_item,
+                    collection_dict,
+                    use_dataset_dates,
                 )
         return None
 
@@ -165,12 +208,12 @@ class CopMarineSearch(StaticStacSearch):
         product_type: str,
         item_key: str,
         s3_url: str,
-        dataset_item: Dict[str, Any],
-        collection_dict: Dict[str, Any],
+        dataset_item: dict[str, Any],
+        collection_dict: dict[str, Any],
         use_dataset_dates: bool = False,
     ) -> Optional[EOProduct]:
 
-        item_id = item_key.split("/")[-1].split(".")[0]
+        item_id = os.path.splitext(item_key.split("/")[-1])[0]
         download_url = s3_url + "/" + item_key
         properties = {
             "id": item_id,
@@ -180,20 +223,16 @@ class CopMarineSearch(StaticStacSearch):
             "dataset": dataset_item["id"],
         }
         if use_dataset_dates:
-            if "start_datetime" in dataset_item:
-                properties["startTimeFromAscendingNode"] = dataset_item[
-                    "start_datetime"
-                ]
-                properties["completionTimeFromAscendingNode"] = dataset_item[
-                    "end_datetime"
-                ]
-            elif "datetime" in dataset_item:
-                properties["startTimeFromAscendingNode"] = dataset_item["datetime"]
-                properties["completionTimeFromAscendingNode"] = dataset_item["datetime"]
+            dates = _get_dates_from_dataset_data(dataset_item)
+            if not dates:
+                return None
+            properties["startTimeFromAscendingNode"] = dates["start"]
+            properties["completionTimeFromAscendingNode"] = dates["end"]
         else:
-            item_dates = re.findall(r"\d{8}", item_key)
+            item_dates = re.findall(r"(\d{4})(0[1-9]|1[0-2])([0-3]\d)", item_id)
             if not item_dates:
-                item_dates = re.findall(r"\d{6}", item_key)
+                item_dates = re.findall(r"_(\d{4})(0[1-9]|1[0-2])", item_id)
+            item_dates = ["".join(row) for row in item_dates]
             item_start = _get_date_from_yyyymmdd(item_dates[0], item_key)
             if not item_start:  # identified pattern was not a valid datetime
                 return None
@@ -209,11 +248,26 @@ class CopMarineSearch(StaticStacSearch):
             ).strftime("%Y-%m-%dT%H:%M:%SZ")
 
         for key, value in collection_dict["properties"].items():
-            if key not in ["id", "title", "start_datetime", "end_datetime"]:
+            if key not in ["id", "title", "start_datetime", "end_datetime", "datetime"]:
                 properties[key] = value
         for key, value in dataset_item["properties"].items():
-            if key not in ["id", "title", "start_datetime", "end_datetime"]:
+            if key not in ["id", "title", "start_datetime", "end_datetime", "datetime"]:
                 properties[key] = value
+
+        code_mapping = self.config.products.get(product_type, {}).get(
+            "code_mapping", None
+        )
+        if code_mapping:
+            id_parts = item_id.split("_")
+            if len(id_parts) > code_mapping["index"]:
+                code = id_parts[code_mapping["index"]]
+                if "pattern" not in code_mapping:
+                    properties[code_mapping["param"]] = code
+                elif re.findall(code_mapping["pattern"], code):
+                    properties[code_mapping["param"]] = re.findall(
+                        code_mapping["pattern"], code
+                    )[0]
+
         _check_int_values_properties(properties)
 
         properties["thumbnail"] = collection_dict["assets"]["thumbnail"]["href"]
@@ -234,7 +288,7 @@ class CopMarineSearch(StaticStacSearch):
         self,
         prep: PreparedSearch = PreparedSearch(),
         **kwargs: Any,
-    ) -> Tuple[List[EOProduct], Optional[int]]:
+    ) -> tuple[list[EOProduct], Optional[int]]:
         """
         Implementation of search for the Copernicus Marine provider
         :param prep: object containing search parameterds
@@ -254,7 +308,7 @@ class CopMarineSearch(StaticStacSearch):
                 "parameter product type is required for search with cop_marine provider"
             )
         collection_dict, datasets_items_list = self._get_product_type_info(product_type)
-        products: List[EOProduct] = []
+        products: list[EOProduct] = []
         start_index = items_per_page * (page - 1) + 1
         num_total = 0
         for i, dataset_item in enumerate(datasets_items_list):
@@ -348,16 +402,54 @@ class CopMarineSearch(StaticStacSearch):
 
                 for obj in s3_objects["Contents"]:
                     item_key = obj["Key"]
+                    item_id = os.path.splitext(item_key.split("/")[-1])[0]
                     # filter according to date(s) in item id
-                    item_dates = re.findall(r"\d{8}", item_key)
+                    item_dates = re.findall(r"(\d{4})(0[1-9]|1[0-2])([0-3]\d)", item_id)
                     if not item_dates:
-                        item_dates = re.findall(r"\d{6}", item_key)
-                    item_start = _get_date_from_yyyymmdd(item_dates[0], item_key)
-                    if not item_start:  # identified pattern was not a valid datetime
+                        item_dates = re.findall(r"_(\d{4})(0[1-9]|1[0-2])", item_id)
+                    item_dates = [
+                        "".join(row) for row in item_dates
+                    ]  # join tuples returned by findall
+                    item_start = None
+                    item_end = None
+                    use_dataset_dates = False
+                    if item_dates:
+                        item_start = _get_date_from_yyyymmdd(item_dates[0], item_key)
+                        if len(item_dates) > 2:  # start, end and created_at timestamps
+                            item_end = _get_date_from_yyyymmdd(item_dates[1], item_key)
+                    if not item_start:
+                        # no valid datetime given in id
+                        use_dataset_dates = True
+                        dates = _get_dates_from_dataset_data(dataset_item)
+                        if dates:
+                            item_start_str = dates["start"].replace("Z", "+0000")
+                            item_end_str = dates["end"].replace("Z", "+0000")
+                            try:
+                                item_start = datetime.strptime(
+                                    item_start_str, "%Y-%m-%dT%H:%M:%S.%f%z"
+                                )
+                                item_end = datetime.strptime(
+                                    item_end_str, "%Y-%m-%dT%H:%M:%S.%f%z"
+                                )
+                            except ValueError:
+                                item_start = datetime.strptime(
+                                    item_start_str, "%Y-%m-%dT%H:%M:%S%z"
+                                )
+                                item_end = datetime.strptime(
+                                    item_end_str, "%Y-%m-%dT%H:%M:%S%z"
+                                )
+                    if not item_start:
+                        # no valid datetime in id and dataset data
                         continue
                     if item_start > end_date:
                         stop_search = True
-                    if not item_dates or (start_date <= item_start <= end_date):
+                    if (
+                        (start_date <= item_start <= end_date)
+                        or (item_end and start_date <= item_end <= end_date)
+                        or (
+                            item_end and item_start < start_date and item_end > end_date
+                        )
+                    ):
                         num_total += 1
                         if num_total < start_index:
                             continue
@@ -368,6 +460,7 @@ class CopMarineSearch(StaticStacSearch):
                                 endpoint_url + "/" + bucket,
                                 dataset_item,
                                 collection_dict,
+                                use_dataset_dates,
                             )
                             if product:
                                 products.append(product)

eodag/plugins/search/creodias_s3.py

@@ -17,26 +17,22 @@
 # limitations under the License.
 import logging
 from types import MethodType
-from typing import Any, List
+from typing import Any
 
-import boto3
-import botocore
 from botocore.exceptions import BotoCoreError
 
-from eodag.api.product import AssetsDict, EOProduct  # type: ignore
+from eodag.api.product import EOProduct  # type: ignore
 from eodag.api.search_result import RawSearchResult
-from eodag.config import PluginConfig
-from eodag.plugins.authentication.aws_auth import AwsAuth
 from eodag.plugins.search.qssearch import ODataV4Search
-from eodag.utils import guess_file_type
-from eodag.utils.exceptions import AuthenticationError, MisconfiguredError, RequestError
+from eodag.utils.exceptions import RequestError
+from eodag.utils.s3 import update_assets_from_s3
 
-DATA_EXTENSIONS = ["jp2", "tiff", "nc", "grib"]
 logger = logging.getLogger("eodag.search.creodiass3")
 
 
 def patched_register_downloader(self, downloader, authenticator):
     """Add the download information to the product.
+
     :param self: product to which information should be added
     :param downloader: The download method that it can use
                       :class:`~eodag.plugins.download.base.Download` or
@@ -48,69 +44,24 @@ def patched_register_downloader(self, downloader, authenticator):
     self.register_downloader_only(downloader, authenticator)
     # and also update assets
     try:
-        _update_assets(self, downloader.config, authenticator)
+        update_assets_from_s3(
+            self, authenticator, getattr(downloader.config, "s3_endpoint", None)
+        )
     except BotoCoreError as e:
-        raise RequestError(f"could not update assets: {str(e)}") from e
-
-
-def _update_assets(product: EOProduct, config: PluginConfig, auth: AwsAuth):
-    product.assets = {}
-    prefix = (
-        product.properties.get("productIdentifier", None).replace("/eodata/", "") + "/"
-    )
-    if prefix:
-        try:
-            auth_dict = auth.authenticate()
-            required_creds = ["aws_access_key_id", "aws_secret_access_key"]
-            if not all(x in auth_dict for x in required_creds):
-                raise MisconfiguredError(
-                    f"Incomplete credentials for {product.provider}, missing "
-                    f"{[x for x in required_creds if x not in auth_dict]}"
-                )
-            if not getattr(auth, "s3_client", None):
-                auth.s3_client = boto3.client(
-                    "s3",
-                    endpoint_url=config.base_uri,
-                    aws_access_key_id=auth_dict["aws_access_key_id"],
-                    aws_secret_access_key=auth_dict["aws_secret_access_key"],
-                )
-            logger.debug("Listing assets in %s", prefix)
-            product.assets = AssetsDict(product)
-            for asset in auth.s3_client.list_objects(
-                Bucket=config.s3_bucket, Prefix=prefix, MaxKeys=300
-            )["Contents"]:
-                asset_basename = (
-                    asset["Key"].split("/")[-1] if "/" in asset["Key"] else asset["Key"]
-                )
-
-                if len(asset_basename) > 0 and asset_basename not in product.assets:
-                    role = (
-                        "data"
-                        if asset_basename.split(".")[-1] in DATA_EXTENSIONS
-                        else "metadata"
-                    )
-
-                    product.assets[asset_basename] = {
-                        "title": asset_basename,
-                        "roles": [role],
-                        "href": f"s3://{config.s3_bucket}/{asset['Key']}",
-                    }
-                    if mime_type := guess_file_type(asset["Key"]):
-                        product.assets[asset_basename]["type"] = mime_type
-            # update driver
-            product.driver = product.get_driver()
-
-        except botocore.exceptions.ClientError as e:
-            if str(auth.config.auth_error_code) in str(e):
-                raise AuthenticationError(
-                    f"Authentication failed on {config.base_uri} s3"
-                ) from e
-            raise RequestError(f"assets for product {prefix} could not be found") from e
+        raise RequestError.from_error(e, "could not update assets") from e
 
 
 class CreodiasS3Search(ODataV4Search):
     """
-    Search on creodias and adapt results to s3
+    ``CreodiasS3Search`` is an extension of :class:`~eodag.plugins.search.qssearch.ODataV4Search`,
+    it executes a Search on creodias and adapts results so that the assets contain links to s3.
+    It has the same configuration parameters as :class:`~eodag.plugins.search.qssearch.ODataV4Search` and
+    one additional parameter:
+
+    :param provider: provider name
+    :param config: Search plugin configuration:
+
+        * :attr:`~eodag.config.PluginConfig.s3_endpoint` (``str``) (**mandatory**): base url of the s3
     """
 
     def __init__(self, provider, config):
@@ -118,7 +69,7 @@ class CreodiasS3Search(ODataV4Search):
 
     def normalize_results(
         self, results: RawSearchResult, **kwargs: Any
-    ) -> List[EOProduct]:
+    ) -> list[EOProduct]:
         """Build EOProducts from provider results"""
 
         products = super(CreodiasS3Search, self).normalize_results(results, **kwargs)

eodag/plugins/search/csw.py

@@ -19,7 +19,7 @@ from __future__ import annotations
 
 import logging
 import re
-from typing import TYPE_CHECKING, Any, Dict, List, Optional, Tuple, Union
+from typing import TYPE_CHECKING, Any, Optional, Union
 
 import pyproj
 from owslib.csw import CatalogueServiceWeb
@@ -52,7 +52,47 @@ SUPPORTED_REFERENCE_SCHEMES = ["WWW:DOWNLOAD-1.0-http--download"]
 
 
 class CSWSearch(Search):
-    """A plugin for implementing search based on OGC CSW"""
+    """A plugin for implementing search based on OGC CSW
+
+    :param provider: provider name
+    :param config: Search plugin configuration:
+
+        * :attr:`~eodag.config.PluginConfig.api_endpoint` (``str``) (**mandatory**): The endpoint of the
+          provider's search interface
+        * :attr:`~eodag.config.PluginConfig.version` (``str``): OGC Catalogue Service version; default: ``2.0.2``
+        * :attr:`~eodag.config.PluginConfig.search_definition` (``dict[str, Any]``) (**mandatory**):
+
+          * **product_type_tags** (``list[dict[str, Any]``): dict of product type tags
+          * **resource_location_filter** (``str``): regex string
+          * **date_tags** (``dict[str, Any]``): tags for start and end
+
+        * :attr:`~eodag.config.PluginConfig.metadata_mapping` (``dict[str, Any]``): The search plugins of this kind can
+          detect when a metadata mapping is "query-able", and get the semantics of how to format the query string
+          parameter that enables to make a query on the corresponding metadata. To make a metadata query-able,
+          just configure it in the metadata mapping to be a list of 2 items, the first one being the
+          specification of the query string search formatting. The later is a string following the
+          specification of Python string formatting, with a special behaviour added to it. For example,
+          an entry in the metadata mapping of this kind::
+
+                completionTimeFromAscendingNode:
+                    - 'f=acquisition.endViewingDate:lte:{completionTimeFromAscendingNode#timestamp}'
+                    - '$.properties.acquisition.endViewingDate'
+
+          means that the search url will have a query string parameter named ``f`` with a value of
+          ``acquisition.endViewingDate:lte:1543922280.0`` if the search was done with the value
+          of ``completionTimeFromAscendingNode`` being ``2018-12-04T12:18:00``. What happened is that
+          ``{completionTimeFromAscendingNode#timestamp}`` was replaced with the timestamp of the value
+          of ``completionTimeFromAscendingNode``. This example shows all there is to know about the
+          semantics of the query string formatting introduced by this plugin: any eodag search parameter
+          can be referenced in the query string with an additional optional conversion function that
+          is separated from it by a ``#`` (see :func:`~eodag.api.product.metadata_mapping.format_metadata` for further
+          details on the available converters). Note that for the values in the
+          :attr:`~eodag.config.PluginConfig.free_text_search_operations` configuration parameter follow the same rule.
+          If the metadata_mapping is not a list but only a string, this means that the parameters is not queryable but
+          it is included in the result obtained from the provider. The string indicates how the provider result should
+          be mapped to the eodag parameter.
+
+    """
 
     def __init__(self, provider: str, config: PluginConfig) -> None:
         super(CSWSearch, self).__init__(provider, config)
@@ -67,7 +107,7 @@ class CSWSearch(Search):
         self,
         prep: PreparedSearch = PreparedSearch(),
         **kwargs: Any,
-    ) -> Tuple[List[EOProduct], Optional[int]]:
+    ) -> tuple[list[EOProduct], Optional[int]]:
         """Perform a search on a OGC/CSW-like interface"""
         product_type = kwargs.get("productType")
         if product_type is None:
@@ -77,7 +117,7 @@ class CSWSearch(Search):
             self.__init_catalog(**getattr(auth.config, "credentials", {}))
         else:
             self.__init_catalog()
-        results: List[EOProduct] = []
+        results: list[EOProduct] = []
         if self.catalog:
             provider_product_type = self.config.products[product_type]["productType"]
             for product_type_def in self.config.search_definition["product_type_tags"]:
@@ -189,12 +229,12 @@ class CSWSearch(Search):
 
     def __convert_query_params(
         self,
-        product_type_def: Dict[str, Any],
+        product_type_def: dict[str, Any],
         product_type: str,
-        params: Dict[str, Any],
-    ) -> Union[List[OgcExpression], List[List[OgcExpression]]]:
+        params: dict[str, Any],
+    ) -> Union[list[OgcExpression], list[list[OgcExpression]]]:
         """Translates eodag search to CSW constraints using owslib constraint classes"""
-        constraints: List[OgcExpression] = []
+        constraints: list[OgcExpression] = []
         # How the match should be performed (fuzzy, prefix, postfix or exact).
         # defaults to fuzzy
         pt_tag, matching = (