eodag 3.2.0__py3-none-any.whl → 3.3.0__py3-none-any.whl

eodag/plugins/search/build_search_result.py

@@ -22,7 +22,8 @@ import hashlib
 import logging
 import re
 from collections import OrderedDict
-from datetime import datetime, timedelta
+from datetime import date, datetime, timedelta, timezone
+from types import MethodType
 from typing import TYPE_CHECKING, Annotated, Any, Optional, Union
 from urllib.parse import quote_plus, unquote_plus
 
@@ -35,19 +36,22 @@ from pydantic import Field
 from pydantic.fields import FieldInfo
 from requests.auth import AuthBase
 from shapely.geometry.base import BaseGeometry
-from typing_extensions import get_args
+from typing_extensions import get_args  # noqa: F401
 
 from eodag.api.product import EOProduct
 from eodag.api.product.metadata_mapping import (
+    DEFAULT_GEOMETRY,
     NOT_AVAILABLE,
     OFFLINE_STATUS,
+    STAGING_STATUS,
     format_metadata,
+    mtd_cfg_as_conversion_and_querypath,
     properties_from_json,
 )
 from eodag.api.search_result import RawSearchResult
 from eodag.plugins.search import PreparedSearch
 from eodag.plugins.search.qssearch import PostJsonSearch, QueryStringSearch
-from eodag.types import json_field_definition_to_python
+from eodag.types import json_field_definition_to_python  # noqa: F401
 from eodag.types.queryables import Queryables, QueryablesDict
 from eodag.utils import (
     DEFAULT_MISSION_START_DATE,
@@ -57,7 +61,7 @@ from eodag.utils import (
     get_geometry_from_various,
     is_range_in_range,
 )
-from eodag.utils.exceptions import ValidationError
+from eodag.utils.exceptions import DownloadError, NotAvailableError, ValidationError
 from eodag.utils.requests import fetch_json
 
 if TYPE_CHECKING:
@@ -286,6 +290,133 @@ def ecmwf_format(v: str) -> str:
     return ECMWF_PREFIX + v if v in ALLOWED_KEYWORDS else v
 
 
+def get_min_max(
+    value: Optional[Union[str, list[str]]] = None,
+) -> tuple[Optional[str], Optional[str]]:
+    """Returns the min and max from a list of strings or the same string if a single string is given."""
+    if isinstance(value, list):
+        sorted_values = sorted(value)
+        return sorted_values[0], sorted_values[-1]
+    return value, value
+
+
+def append_time(input_date: date, time: Optional[str]) -> datetime:
+    """
+    Parses a time string in format HHMM and appends it to a date.
+
+    if the time string is in format HH:MM we convert it to HHMM
+    """
+    if not time:
+        time = "0000"
+    time = time.replace(":", "")
+    if time == "2400":
+        time = "0000"
+    dt = datetime.combine(input_date, datetime.strptime(time, "%H%M").time())
+    dt.replace(tzinfo=timezone.utc)
+    return dt
+
+
+def parse_date(
+    date_str: str, time: Optional[Union[str, list[str]]]
+) -> tuple[datetime, datetime]:
+    """Parses a date string in formats YYYY-MM-DD, YYYMMDD, solo or in start/end or start/to/end intervals."""
+    if "to" in date_str:
+        start_date_str, end_date_str = date_str.split("/to/")
+    elif "/" in date_str:
+        dates = date_str.split("/")
+        start_date_str = dates[0]
+        end_date_str = dates[-1]
+    else:
+        start_date_str = end_date_str = date_str
+
+    # Update YYYYMMDD formatted dates
+    if re.match(r"^\d{8}$", start_date_str):
+        start_date_str = (
+            f"{start_date_str[:4]}-{start_date_str[4:6]}-{start_date_str[6:]}"
+        )
+    if re.match(r"^\d{8}$", end_date_str):
+        end_date_str = f"{end_date_str[:4]}-{end_date_str[4:6]}-{end_date_str[6:]}"
+
+    start_date = datetime.fromisoformat(start_date_str.rstrip("Z"))
+    end_date = datetime.fromisoformat(end_date_str.rstrip("Z"))
+
+    if time:
+        start_t, end_t = get_min_max(time)
+        start_date = append_time(start_date.date(), start_t)
+        end_date = append_time(end_date.date(), end_t)
+
+    return start_date, end_date
+
+
+def parse_year_month_day(
+    year: Union[str, list[str]],
+    month: Optional[Union[str, list[str]]] = None,
+    day: Optional[Union[str, list[str]]] = None,
+    time: Optional[Union[str, list[str]]] = None,
+) -> tuple[datetime, datetime]:
+    """Extracts and returns the year, month, day, and time from the parameters."""
+
+    def build_date(year, month=None, day=None, time=None) -> datetime:
+        """Datetime from default_date with updated year, month, day and time."""
+        updated_date = datetime(int(year), 1, 1).replace(
+            month=int(month) if month is not None else 1,
+            day=int(day) if day is not None else 1,
+        )
+        if time is not None:
+            updated_date = append_time(updated_date.date(), time)
+        return updated_date
+
+    start_y, end_y = get_min_max(year)
+    start_m, end_m = get_min_max(month)
+    start_d, end_d = get_min_max(day)
+    start_t, end_t = get_min_max(time)
+
+    start_date = build_date(start_y, start_m, start_d, start_t)
+    end_date = build_date(end_y, end_m, end_d, end_t)
+
+    return start_date, end_date
+
+
+def ecmwf_temporal_to_eodag(
+    params: dict[str, Any]
+) -> tuple[Optional[str], Optional[str]]:
+    """
+    Converts ECMWF temporal parameters to EODAG temporal parameters.
+
+    ECMWF temporal parameters:
+        - **year** or **hyear**: Union[str, list[str]] — Year(s) as a string or list of strings.
+        - **month** or **hmonth**: Union[str, list[str]] — Month(s) as a string or list of strings.
+        - **day** or **hday**: Union[str, list[str]] — Day(s) as a string or list of strings.
+        - **time**: str — A string representing the time in the format `HHMM` (e.g., `0200`, `0800`, `1400`).
+        - **date**: str — A string in one of the formats:
+            - `YYYY-MM-DD`
+            - `YYYY-MM-DD/YYYY-MM-DD`
+            - `YYYY-MM-DD/to/YYYY-MM-DD`
+
+    :param params: Dictionary containing ECMWF temporal parameters.
+    :return: A tuple with:
+        - **start**: A string in the format `YYYY-MM-DDTHH:MM:SSZ`.
+        - **end**: A string in the format `YYYY-MM-DDTHH:MM:SSZ`.
+    """
+    start = end = None
+
+    if date := params.get("date"):
+        start, end = parse_date(date, params.get("time"))
+
+    elif year := params.get("year") or params.get("hyear"):
+        year = params.get("year") or params.get("hyear")
+        month = params.get("month") or params.get("hmonth")
+        day = params.get("day") or params.get("hday")
+        time = params.get("time")
+
+        start, end = parse_year_month_day(year, month, day, time)
+
+    if start and end:
+        return start.strftime("%Y-%m-%dT%H:%M:%SZ"), end.strftime("%Y-%m-%dT%H:%M:%SZ")
+    else:
+        return None, None
+
+
 class ECMWFSearch(PostJsonSearch):
     """ECMWF search plugin.
 
@@ -335,6 +466,16 @@ class ECMWFSearch(PostJsonSearch):
         self.config.__dict__.setdefault("api_endpoint", "")
         self.config.pagination.setdefault("next_page_query_obj", "{{}}")
 
+        # defaut conf for accepting custom query params
+        self.config.__dict__.setdefault(
+            "discover_metadata",
+            {
+                "auto_discovery": True,
+                "search_param": "{metadata}",
+                "metadata_pattern": "^[a-zA-Z0-9][a-zA-Z0-9_]*$",
+            },
+        )
+
     def do_search(self, *args: Any, **kwargs: Any) -> list[dict[str, Any]]:
         """Should perform the actual search request.
 
@@ -497,22 +638,10 @@ class ECMWFSearch(PostJsonSearch):
             if not isinstance(mapping, list):
                 mapping = product_type_conf[END]
             if isinstance(mapping, list):
-                # get time parameters (date, year, month, ...) from metadata mapping
-                input_mapping = mapping[0].replace("{{", "").replace("}}", "")
-                time_params = [
-                    values.split(":")[0].strip() for values in input_mapping.split(",")
-                ]
-                time_params = [
-                    tp.replace('"', "").replace("'", "") for tp in time_params
-                ]
                 # if startTime is not given but other time params (e.g. year/month/(day)) are given,
                 # no default date is required
-                in_keywords = True
-                for tp in time_params:
-                    if tp not in keywords:
-                        in_keywords = False
-                        break
-                if not in_keywords:
+                start, end = ecmwf_temporal_to_eodag(keywords)
+                if start is None:
                     keywords[START] = self.get_product_type_cfg_value(
                         "missionStartDate", DEFAULT_MISSION_START_DATE
                     )
@@ -912,7 +1041,7 @@ class ECMWFSearch(PostJsonSearch):
         """
         # Rename keywords from form with metadata mapping.
         # Needed to map constraints like "xxxx" to eodag parameter "ecmwf:xxxx"
-        required = [ecmwf_format(k) for k in required_keywords]
+        required = [ecmwf_format(k) for k in required_keywords]  # noqa: F841
 
         queryables: dict[str, Annotated[Any, FieldInfo]] = {}
         for name, values in available_values.items():
@@ -998,92 +1127,74 @@ class ECMWFSearch(PostJsonSearch):
         _dc_qs = kwargs.pop("_dc_qs", None)
         if _dc_qs is not None:
             qs = unquote_plus(unquote_plus(_dc_qs))
-            sorted_unpaginated_query_params = geojson.loads(qs)
+            sorted_unpaginated_qp = geojson.loads(qs)
         else:
-            # update result with query parameters without pagination (or search-only params)
-            if isinstance(
-                self.config.pagination["next_page_query_obj"], str
-            ) and hasattr(results, "query_params_unpaginated"):
-                unpaginated_query_params = results.query_params_unpaginated
-            elif isinstance(self.config.pagination["next_page_query_obj"], str):
-                next_page_query_obj = orjson.loads(
-                    self.config.pagination["next_page_query_obj"].format()
-                )
-                unpaginated_query_params = {
-                    k: v
-                    for k, v in results.query_params.items()
-                    if (k, v) not in next_page_query_obj.items()
-                }
-            else:
-                unpaginated_query_params = self.query_params
-            # query hash, will be used to build a product id
-            sorted_unpaginated_query_params = dict_items_recursive_sort(
-                unpaginated_query_params
-            )
-
-        # use all available query_params to parse properties
-        result = dict(
-            result,
-            **sorted_unpaginated_query_params,
-            qs=sorted_unpaginated_query_params,
-        )
+            sorted_unpaginated_qp = dict_items_recursive_sort(results.query_params)
 
         # remove unwanted query params
         for param in getattr(self.config, "remove_from_query", []):
-            sorted_unpaginated_query_params.pop(param, None)
+            sorted_unpaginated_qp.pop(param, None)
 
-        qs = geojson.dumps(sorted_unpaginated_query_params)
+        if result:
+            properties = result
+            properties.update(result.pop("request_params", None) or {})
 
-        query_hash = hashlib.sha1(str(qs).encode("UTF-8")).hexdigest()
+            properties = {k: v for k, v in properties.items() if not k.startswith("__")}
 
-        # update result with product_type_def_params and search args if not None (and not auth)
-        kwargs.pop("auth", None)
-        result.update(results.product_type_def_params)
-        result = dict(result, **{k: v for k, v in kwargs.items() if v is not None})
+            properties["geometry"] = properties.get("area") or DEFAULT_GEOMETRY
 
-        # parse properties
-        parsed_properties = properties_from_json(
-            result,
-            self.config.metadata_mapping,
-            discovery_config=getattr(self.config, "discover_metadata", {}),
-        )
+            start, end = ecmwf_temporal_to_eodag(properties)
+            properties["startTimeFromAscendingNode"] = start
+            properties["completionTimeFromAscendingNode"] = end
 
-        properties = {
-            # use product_type_config as default properties
-            **getattr(self.config, "product_type_config", {}),
-            **{ecmwf_format(k): v for k, v in parsed_properties.items()},
-        }
-
-        def slugify(date_str: str) -> str:
-            return date_str.split("T")[0].replace("-", "")
+        else:
+            # use all available query_params to parse properties
+            result_data: dict[str, Any] = {
+                **results.product_type_def_params,
+                **sorted_unpaginated_qp,
+                **{"qs": sorted_unpaginated_qp},
+            }
 
-        # build product id
-        product_id = (product_type or kwargs.get("dataset") or self.provider).upper()
+            # update result with product_type_def_params and search args if not None (and not auth)
+            kwargs.pop("auth", None)
+            result_data.update(results.product_type_def_params)
+            result_data = {
+                **result_data,
+                **{k: v for k, v in kwargs.items() if v is not None},
+            }
 
-        start = properties.get(START, NOT_AVAILABLE)
-        end = properties.get(END, NOT_AVAILABLE)
+            properties = properties_from_json(
+                result_data,
+                self.config.metadata_mapping,
+                discovery_config=getattr(self.config, "discover_metadata", {}),
+            )
 
-        if start != NOT_AVAILABLE:
-            product_id += f"_{slugify(start)}"
-            if end != NOT_AVAILABLE:
-                product_id += f"_{slugify(end)}"
+            query_hash = hashlib.sha1(str(result_data).encode("UTF-8")).hexdigest()
 
-        product_id += f"_{query_hash}"
+            properties["title"] = properties["id"] = (
+                (product_type or kwargs.get("dataset", self.provider)).upper()
+                + "_ORDERABLE_"
+                + query_hash
+            )
 
-        properties["id"] = properties["title"] = product_id
+        qs = geojson.dumps(sorted_unpaginated_qp)
 
         # used by server mode to generate downloadlink href
+        # TODO: to remove once the legacy server is removed
         properties["_dc_qs"] = quote_plus(qs)
 
         product = EOProduct(
             provider=self.provider,
-            productType=product_type,
-            properties=properties,
+            properties={ecmwf_format(k): v for k, v in properties.items()},
+            **kwargs,
         )
 
-        return [
-            product,
-        ]
+        # backup original register_downloader to register_downloader_only
+        product.register_downloader_only = product.register_downloader
+        # patched register_downloader that will also update properties
+        product.register_downloader = MethodType(patched_register_downloader, product)
+
+        return [product]
 
     def count_hits(
         self, count_url: Optional[str] = None, result_type: Optional[str] = None
@@ -1097,6 +1208,83 @@ class ECMWFSearch(PostJsonSearch):
         return 1
 
 
+def _check_id(product: EOProduct) -> EOProduct:
+    """Check if the id is the one of an existing job.
+
+    If the job exists, poll it, otherwise, raise an error.
+
+    :param product: The product to check the id for
+    :raises: :class:`~eodag.utils.exceptions.ValidationError`
+    """
+    if not (product_id := product.search_kwargs.get("id")):
+        return product
+
+    on_response_mm = getattr(product.downloader.config, "order_on_response", {}).get(
+        "metadata_mapping", {}
+    )
+    if not on_response_mm:
+        return product
+
+    logger.debug(f"Update product properties using given orderId {product_id}")
+    on_response_mm_jsonpath = mtd_cfg_as_conversion_and_querypath(
+        on_response_mm,
+    )
+    properties_update = properties_from_json(
+        {}, {**on_response_mm_jsonpath, **{"orderId": (None, product_id)}}
+    )
+    product.properties.update(
+        {k: v for k, v in properties_update.items() if v != NOT_AVAILABLE}
+    )
+
+    auth = product.downloader_auth.authenticate() if product.downloader_auth else None
+
+    # try to poll the job corresponding to the given id
+    try:
+        product.downloader._order_status(product=product, auth=auth)  # type: ignore
+    # when a NotAvailableError is catched, it means the product is not ready and still needs to be polled
+    except NotAvailableError:
+        product.properties["storageStatus"] = STAGING_STATUS
+    except Exception as e:
+        if (
+            isinstance(e, DownloadError) or isinstance(e, ValidationError)
+        ) and "order status could not be checked" in e.args[0]:
+            raise ValidationError(
+                f"Item {product_id} does not exist with {product.provider}."
+            ) from e
+        raise ValidationError(e.args[0]) from e
+
+    # update product id
+    product.properties["id"] = product_id
+    # update product type if needed
+    if product.product_type is None:
+        product.product_type = product.properties.get("ecmwf:dataset")
+    # update product title
+    product.properties["title"] = (
+        (product.product_type or product.provider).upper() + "_" + product_id
+    )
+    # use NOT_AVAILABLE as fallback product_type to avoid using guess_product_type
+    if product.product_type is None:
+        product.product_type = NOT_AVAILABLE
+
+    return product
+
+
+def patched_register_downloader(self, downloader, authenticator):
+    """Register product donwloader and update properties if searched by id.
+
+    :param self: product to which information should be added
+    :param downloader: The download method that it can use
+                    :class:`~eodag.plugins.download.base.Download` or
+                    :class:`~eodag.plugins.api.base.Api`
+    :param authenticator: The authentication method needed to perform the download
+                        :class:`~eodag.plugins.authentication.base.Authentication`
+    """
+    # register downloader
+    self.register_downloader_only(downloader, authenticator)
+    # and also update properties
+    _check_id(self)
+
+
 class MeteoblueSearch(ECMWFSearch):
     """MeteoblueSearch search plugin.
 
@@ -1165,6 +1353,97 @@ class MeteoblueSearch(ECMWFSearch):
         """
         return QueryStringSearch.build_query_string(self, product_type, query_dict)
 
+    def normalize_results(self, results, **kwargs):
+        """Build :class:`~eodag.api.product._product.EOProduct` from provider result
+
+        :param results: Raw provider result as single dict in list
+        :param kwargs: Search arguments
+        :returns: list of single :class:`~eodag.api.product._product.EOProduct`
+        """
+
+        product_type = kwargs.get("productType")
+
+        result = results[0]
+
+        # datacube query string got from previous search
+        _dc_qs = kwargs.pop("_dc_qs", None)
+        if _dc_qs is not None:
+            qs = unquote_plus(unquote_plus(_dc_qs))
+            sorted_unpaginated_query_params = geojson.loads(qs)
+        else:
+            next_page_query_obj = orjson.loads(
+                self.config.pagination["next_page_query_obj"].format()
+            )
+            unpaginated_query_params = {
+                k: v
+                for k, v in results.query_params.items()
+                if (k, v) not in next_page_query_obj.items()
+            }
+            # query hash, will be used to build a product id
+            sorted_unpaginated_query_params = dict_items_recursive_sort(
+                unpaginated_query_params
+            )
+
+        # use all available query_params to parse properties
+        result = dict(
+            result,
+            **sorted_unpaginated_query_params,
+            qs=sorted_unpaginated_query_params,
+        )
+
+        qs = geojson.dumps(sorted_unpaginated_query_params)
+
+        query_hash = hashlib.sha1(str(qs).encode("UTF-8")).hexdigest()
+
+        # update result with product_type_def_params and search args if not None (and not auth)
+        kwargs.pop("auth", None)
+        result.update(results.product_type_def_params)
+        result = dict(result, **{k: v for k, v in kwargs.items() if v is not None})
+
+        # parse properties
+        parsed_properties = properties_from_json(
+            result,
+            self.config.metadata_mapping,
+            discovery_config=getattr(self.config, "discover_metadata", {}),
+        )
+
+        properties = {
+            # use product_type_config as default properties
+            **getattr(self.config, "product_type_config", {}),
+            **{ecmwf_format(k): v for k, v in parsed_properties.items()},
+        }
+
+        def slugify(date_str: str) -> str:
+            return date_str.split("T")[0].replace("-", "")
+
+        # build product id
+        product_id = (product_type or self.provider).upper()
+
+        start = properties.get(START, NOT_AVAILABLE)
+        end = properties.get(END, NOT_AVAILABLE)
+
+        if start != NOT_AVAILABLE:
+            product_id += f"_{slugify(start)}"
+            if end != NOT_AVAILABLE:
+                product_id += f"_{slugify(end)}"
+
+        product_id += f"_{query_hash}"
+
+        properties["id"] = properties["title"] = product_id
+
+        # used by server mode to generate downloadlink href
+        properties["_dc_qs"] = quote_plus(qs)
+
+        product = EOProduct(
+            provider=self.provider,
+            productType=product_type,
+            properties=properties,
+        )
+
+        return [
+            product,
+        ]
+
 
 class WekeoECMWFSearch(ECMWFSearch):
     """
@@ -1201,6 +1480,10 @@ class WekeoECMWFSearch(ECMWFSearch):
         :returns: list of single :class:`~eodag.api.product._product.EOProduct`
         """
 
+        if kwargs.get("id") and "ORDERABLE" not in kwargs["id"]:
+            # id is order id (only letters and numbers) -> use parent normalize results
+            return super().normalize_results(results, **kwargs)
+
         # formating of orderLink requires access to the productType value.
         results.data = [
             {**result, **results.product_type_def_params} for result in results
@@ -1211,12 +1494,28 @@ class WekeoECMWFSearch(ECMWFSearch):
         if not normalized:
             return normalized
 
-        query_params_encoded = quote_plus(orjson.dumps(results.query_params))
+        # remove unwanted query params
+        excluded_query_params = getattr(self.config, "remove_from_query", [])
+        filtered_query_params = {
+            k: v
+            for k, v in results.query_params.items()
+            if k not in excluded_query_params
+        }
         for product in normalized:
             properties = {**product.properties, **results.query_params}
-            properties["_dc_qs"] = query_params_encoded
+            properties["_dc_qs"] = quote_plus(orjson.dumps(filtered_query_params))
             product.properties = {ecmwf_format(k): v for k, v in properties.items()}
 
+            # update product and title the same way as in parent class
+            splitted_id = product.properties.get("title", "").split("-")
+            dataset = "_".join(splitted_id[:-1])
+            query_hash = splitted_id[-1]
+            product.properties["title"] = product.properties["id"] = (
+                (product.product_type or dataset or self.provider).upper()
+                + "_ORDERABLE_"
+                + query_hash
+            )
+
         return normalized
 
     def do_search(self, *args: Any, **kwargs: Any) -> list[dict[str, Any]]:
@@ -1226,4 +1525,9 @@ class WekeoECMWFSearch(ECMWFSearch):
         :param kwargs: keyword arguments to be used in the search
         :return: list containing the results from the provider in json format
         """
-        return QueryStringSearch.do_search(self, *args, **kwargs)
+        if "id" in kwargs and "ORDERABLE" not in kwargs["id"]:
+            # id is order id (only letters and numbers) -> use parent normalize results.
+            # No real search. We fake it all, then check order status using given id
+            return [{}]
+        else:
+            return QueryStringSearch.do_search(self, *args, **kwargs)