eodag 3.0.0b3__py3-none-any.whl → 3.1.0b1__py3-none-any.whl

eodag/api/product/_assets.py

@@ -98,12 +98,12 @@ class AssetsDict(UserDict):
                 <details><summary style='color: grey;'>
                     <span style='color: black'>'{k}'</span>:&ensp;
                     {{
-                        {"'roles': '<span style='color: black'>"+str(v['roles'])+"</span>',&ensp;"
-                            if v.get("roles") else ""}
-                        {"'type': '"+str(v['type'])+"',&ensp;"
-                            if v.get("type") else ""}
-                        {"'title': '<span style='color: black'>"+str(v['title'])+"</span>',&ensp;"
-                            if v.get("title") else ""}
+                        {"'roles': '<span style='color: black'>" + str(v['roles']) + "</span>',&ensp;"
+                         if v.get("roles") else ""}
+                        {"'type': '" + str(v['type']) + "',&ensp;"
+                         if v.get("type") else ""}
+                        {"'title': '<span style='color: black'>" + str(v['title']) + "</span>',&ensp;"
+                         if v.get("title") else ""}
                         ...
                     }}
                 </summary>

eodag/api/product/_product.py

@@ -116,6 +116,7 @@ class EOProduct:
     properties: Dict[str, Any]
     product_type: Optional[str]
     location: str
+    filename: str
     remote_location: str
     search_kwargs: Any
     geometry: BaseGeometry
@@ -281,8 +282,8 @@ class EOProduct:
     def download(
         self,
         progress_callback: Optional[ProgressCallback] = None,
-        wait: int = DEFAULT_DOWNLOAD_WAIT,
-        timeout: int = DEFAULT_DOWNLOAD_TIMEOUT,
+        wait: float = DEFAULT_DOWNLOAD_WAIT,
+        timeout: float = DEFAULT_DOWNLOAD_TIMEOUT,
         **kwargs: Unpack[DownloadConf],
     ) -> str:
         """Download the EO product using the provided download plugin and the
@@ -525,22 +526,21 @@ class EOProduct:
                 <tr style='background-color: transparent;'>
                     <td style='text-align: left; vertical-align: top;'>
                         {dict_to_html_table({
-                            "provider": self.provider,
-                            "product_type": self.product_type,
-                            "properties[&quot;id&quot;]": self.properties.get('id', None),
-                            "properties[&quot;startTimeFromAscendingNode&quot;]": self.properties.get(
-                                'startTimeFromAscendingNode', None
-                            ),
-                            "properties[&quot;completionTimeFromAscendingNode&quot;]": self.properties.get(
-                                'completionTimeFromAscendingNode', None
-                            ),
-                        }, brackets=False)}
-                        <details><summary style='color: grey; margin-top: 10px;'>properties:&ensp;({
-                            len(self.properties)
-                        })</summary>{dict_to_html_table(self.properties, depth=1)}</details>
-                        <details><summary style='color: grey; margin-top: 10px;'>assets:&ensp;({
-                            len(self.assets)
-                        })</summary>{self.assets._repr_html_(embeded=True)}</details>
+                         "provider": self.provider,
+                         "product_type": self.product_type,
+                         "properties[&quot;id&quot;]": self.properties.get('id', None),
+                         "properties[&quot;startTimeFromAscendingNode&quot;]": self.properties.get(
+                             'startTimeFromAscendingNode', None
+                         ),
+                         "properties[&quot;completionTimeFromAscendingNode&quot;]": self.properties.get(
+                             'completionTimeFromAscendingNode', None
+                         ),
+                         }, brackets=False)}
+                        <details><summary style='color: grey; margin-top: 10px;'>properties:&ensp;({len(
+                             self.properties)})</summary>{
+                                 dict_to_html_table(self.properties, depth=1)}</details>
+                        <details><summary style='color: grey; margin-top: 10px;'>assets:&ensp;({len(
+                                     self.assets)})</summary>{self.assets._repr_html_(embeded=True)}</details>
                     </td>
                     <td {geom_style} title='geometry'>geometry<br />{self.geometry._repr_svg_()}</td>
                     <td {thumbnail_style} title='properties[&quot;thumbnail&quot;]'>{thumbnail_html}</td>

eodag/api/product/metadata_mapping.py

@@ -40,7 +40,9 @@ from typing import (
 import geojson
 import orjson
 import pyproj
+import shapely
 from dateutil.parser import isoparse
+from dateutil.relativedelta import relativedelta
 from dateutil.tz import UTC, tzutc
 from jsonpath_ng.jsonpath import Child, JSONPath
 from lxml import etree
@@ -152,7 +154,7 @@ def get_search_param(map_value: List[str]) -> str:
 
 
 def format_metadata(search_param: str, *args: Any, **kwargs: Any) -> str:
-    """Format a string of form {<field_name>#<conversion_function>}
+    """Format a string of form ``{<field_name>#<conversion_function>}``
 
     The currently understood converters are:
         - ``datetime_to_timestamp_milliseconds``: converts a utc date string to a timestamp in
@@ -178,6 +180,8 @@ def format_metadata(search_param: str, *args: Any, **kwargs: Any) -> str:
         - ``recursive_sub_str``: recursively substitue in the structure (e.g. dict)
           values matching a regex
         - ``slice_str``: slice a string (equivalent to s[start, end, step])
+        - ``to_lower``: Convert a string to lowercase
+        - ``to_upper``: Convert a string to uppercase
         - ``fake_l2a_title_from_l1c``: used to generate SAFE format metadata for data from AWS
         - ``s2msil2a_title_to_aws_productinfo``: used to generate SAFE format metadata for data from AWS
         - ``split_cop_dem_id``: get the bbox by splitting the product id
@@ -362,6 +366,8 @@ def format_metadata(search_param: str, *args: Any, **kwargs: Any) -> str:
 
         @staticmethod
         def convert_to_nwse_bounds(input_geom: BaseGeometry) -> List[float]:
+            if isinstance(input_geom, str):
+                input_geom = shapely.wkt.loads(input_geom)
             return list(input_geom.bounds[-1:] + input_geom.bounds[:-1])
 
         @staticmethod
@@ -557,6 +563,16 @@ def format_metadata(search_param: str, *args: Any, **kwargs: Any) -> str:
             ]
             return string[cmin:cmax:cstep]
 
+        @staticmethod
+        def convert_to_lower(string: str) -> str:
+            """Convert a string to lowercase."""
+            return string.lower()
+
+        @staticmethod
+        def convert_to_upper(string: str) -> str:
+            """Convert a string to uppercase."""
+            return string.upper()
+
         @staticmethod
         def convert_fake_l2a_title_from_l1c(string: str) -> str:
             id_regex = re.compile(
@@ -831,9 +847,9 @@ def format_metadata(search_param: str, *args: Any, **kwargs: Any) -> str:
         def convert_get_hydrological_year(date: str):
             utc_date = MetadataFormatter.convert_to_iso_utc_datetime(date)
             date_object = datetime.strptime(utc_date, "%Y-%m-%dT%H:%M:%S.%fZ")
-            date_object_second_year = date_object + timedelta(days=365)
+            date_object_second_year = date_object + relativedelta(years=1)
             return [
-                f'{date_object.strftime("%Y")}_{date_object_second_year.strftime("%y")}'
+                f"{date_object.strftime('%Y')}_{date_object_second_year.strftime('%y')}"
             ]
 
         @staticmethod
@@ -901,10 +917,8 @@ def format_metadata(search_param: str, *args: Any, **kwargs: Any) -> str:
             return assets_dict
 
     # if stac extension colon separator `:` is in search params, parse it to prevent issues with vformat
-    if re.search(r"{[a-zA-Z0-9_-]*:[a-zA-Z0-9_-]*}", search_param):
-        search_param = re.sub(
-            r"{([a-zA-Z0-9_-]*):([a-zA-Z0-9_-]*)}", r"{\1_COLON_\2}", search_param
-        )
+    if re.search(r"{[\w-]*:[\w#-]*}", search_param):
+        search_param = re.sub(r"{([\w-]*):([\w#-]*)}", r"{\1_COLON_\2}", search_param)
         kwargs = {k.replace(":", "_COLON_"): v for k, v in kwargs.items()}
 
     return MetadataFormatter().vformat(search_param, args, kwargs)
@@ -974,10 +988,24 @@ def properties_from_json(
                     if re.search(r"({[^{}:]+})+", conversion_or_none):
                         conversion_or_none = conversion_or_none.format(**properties)
 
-                    properties[metadata] = format_metadata(
-                        "{%s%s%s}" % (metadata, SEP, conversion_or_none),
-                        **{metadata: extracted_value},
-                    )
+                    if extracted_value == NOT_AVAILABLE:
+                        # try if value can be formatted even if it is not available
+                        try:
+                            properties[metadata] = format_metadata(
+                                "{%s%s%s}" % (metadata, SEP, conversion_or_none),
+                                **{metadata: extracted_value},
+                            )
+                        except ValueError:
+                            logger.debug(
+                                f"{metadata}: {extracted_value} could not be formatted with {conversion_or_none}"
+                            )
+                            continue
+                    else:
+                        # in this case formatting should work, otherwise something is wrong in the mapping
+                        properties[metadata] = format_metadata(
+                            "{%s%s%s}" % (metadata, SEP, conversion_or_none),
+                            **{metadata: extracted_value},
+                        )
         # properties as python objects when possible (format_metadata returns only strings)
         try:
             properties[metadata] = ast.literal_eval(properties[metadata])
@@ -1208,7 +1236,7 @@ def mtd_cfg_as_conversion_and_querypath(
     dest_dict: Dict[str, Any] = {},
     result_type: str = "json",
 ) -> Dict[str, Any]:
-    """Metadata configuration dictionary to querypath with conversion dictionnary
+    """Metadata configuration dictionary to querypath with conversion dictionary
     Transform every src_dict value from jsonpath_str to tuple `(conversion, jsonpath_object)`
     or from xpath_str to tuple `(conversion, xpath_str)`
 
@@ -1461,7 +1489,15 @@ def get_queryable_from_provider(
     :param metadata_mapping: metadata-mapping configuration
     :returns: EODAG configured queryable parameter or None
     """
-    pattern = rf"\b{provider_queryable}\b"
+    pattern = rf"\"{provider_queryable}\""
+    # if 1:1 mapping exists privilege this one instead of other mapping
+    # e.g. provider queryable = year -> use year and not date in which year also appears
+    mapping_values = [
+        v[0] if isinstance(v, list) else "" for v in metadata_mapping.values()
+    ]
+    if provider_queryable in mapping_values:
+        ind = mapping_values.index(provider_queryable)
+        return Queryables.get_queryable_from_alias(list(metadata_mapping.keys())[ind])
     for param, param_conf in metadata_mapping.items():
         if isinstance(param_conf, list) and re.search(pattern, param_conf[0]):
             return Queryables.get_queryable_from_alias(param)
@@ -1489,7 +1525,8 @@ def get_provider_queryable_key(
     provider_queryables: Dict[str, Any],
     metadata_mapping: Dict[str, Union[List[Any], str]],
 ) -> str:
-    """finds the provider queryable corresponding to the given eodag key based on the metadata mapping
+    """Finds the provider queryable corresponding to the given eodag key based on the metadata mapping
+
     :param eodag_key: key in eodag
     :param provider_queryables: queryables returned from the provider
     :param metadata_mapping: metadata mapping from which the keys are retrieved

eodag/api/search_result.py

@@ -18,9 +18,20 @@
 from __future__ import annotations
 
 from collections import UserList
-from typing import TYPE_CHECKING, Any, Dict, List, Optional, Union
+from typing import (
+    TYPE_CHECKING,
+    Annotated,
+    Any,
+    Dict,
+    Iterable,
+    List,
+    Optional,
+    Tuple,
+    Union,
+)
 
 from shapely.geometry import GeometryCollection, shape
+from typing_extensions import Doc
 
 from eodag.api.product import EOProduct
 from eodag.plugins.crunch.filter_date import FilterDate
@@ -47,11 +58,19 @@ class SearchResult(UserList):
 
     data: List[EOProduct]
 
+    errors: Annotated[
+        List[Tuple[str, Exception]], Doc("Tuple of provider name, exception")
+    ]
+
     def __init__(
-        self, products: List[EOProduct], number_matched: Optional[int] = None
+        self,
+        products: List[EOProduct],
+        number_matched: Optional[int] = None,
+        errors: List[Tuple[str, Exception]] = [],
     ) -> None:
-        super(SearchResult, self).__init__(products)
+        super().__init__(products)
         self.number_matched = number_matched
+        self.errors = errors
 
     def crunch(self, cruncher: Crunch, **search_params: Any) -> SearchResult:
         """Do some crunching with the underlying EO products.
@@ -197,6 +216,13 @@ class SearchResult(UserList):
             + "</table>"
         )
 
+    def extend(self, other: Iterable) -> None:
+        """override extend method to include errors"""
+        if isinstance(other, SearchResult):
+            self.errors.extend(other.errors)
+
+        return super().extend(other)
+
 
 class RawSearchResult(UserList):
     """An object representing a collection of raw/unparsed search results obtained from a provider.

eodag/cli.py

@@ -39,6 +39,7 @@ Commands:
 
   noqa: D103
 """
+
 from __future__ import annotations
 
 import json
@@ -56,6 +57,11 @@ from eodag.utils import DEFAULT_ITEMS_PER_PAGE, DEFAULT_PAGE, parse_qs
 from eodag.utils.exceptions import NoMatchingProductType, UnsupportedProvider
 from eodag.utils.logging import setup_logging
 
+try:
+    from eodag.rest.utils import LIVENESS_PROBE_PATH
+except ImportError:
+    pass
+
 if TYPE_CHECKING:
     from click import Context
 
@@ -69,6 +75,18 @@ CRUNCHERS = [
 ]
 
 
+class LivenessFilter:
+    """
+    Filter out requests to the liveness probe endpoint
+    """
+
+    def filter(self, record):
+        """
+        Filter method required by the Python logging API.
+        """
+        return LIVENESS_PROBE_PATH not in record.getMessage()
+
+
 class MutuallyExclusiveOption(click.Option):
     """Mutually Exclusive Options for Click
     from https://gist.github.com/jacobtolar/fb80d5552a9a9dfc32b12a829fa21c0c
@@ -553,12 +571,13 @@ def download(ctx: Context, **kwargs: Any) -> None:
 
         for idx, product in enumerate(search_results):
             if product.downloader is None:
+                downloader = satim_api._plugins_manager.get_download_plugin(product)
                 auth = product.downloader_auth
                 if auth is None:
-                    auth = satim_api._plugins_manager.get_auth_plugin(product.provider)
-                search_results[idx].register_downloader(
-                    satim_api._plugins_manager.get_download_plugin(product), auth
-                )
+                    auth = satim_api._plugins_manager.get_auth_plugin(
+                        downloader, product
+                    )
+                search_results[idx].register_downloader(downloader, auth)
 
             downloaded_file = product.get_quicklook()
             if not downloaded_file:
@@ -573,12 +592,13 @@ def download(ctx: Context, **kwargs: Any) -> None:
         # register downloader
         for idx, product in enumerate(search_results):
             if product.downloader is None:
+                downloader = satim_api._plugins_manager.get_download_plugin(product)
                 auth = product.downloader_auth
                 if auth is None:
-                    auth = satim_api._plugins_manager.get_auth_plugin(product.provider)
-                search_results[idx].register_downloader(
-                    satim_api._plugins_manager.get_download_plugin(product), auth
-                )
+                    auth = satim_api._plugins_manager.get_auth_plugin(
+                        downloader, product
+                    )
+                search_results[idx].register_downloader(downloader, auth)
 
         downloaded_files = satim_api.download_all(search_results)
         if downloaded_files and len(downloaded_files) > 0:
@@ -676,7 +696,9 @@ def serve_rest(
         try:
             pid = os.fork()
         except OSError as e:
-            raise Exception("%s [%d]" % (e.strerror, e.errno))
+            raise Exception(
+                "%s [%d]" % (e.strerror, e.errno) if e.errno is not None else e.strerror
+            )
 
         if pid == 0:
             os.setsid()
@@ -684,19 +706,34 @@ def serve_rest(
         else:
             sys.exit(0)
     else:
+        import logging
+
         logging_config = uvicorn.config.LOGGING_CONFIG
-        if debug:
-            logging_config["loggers"]["uvicorn"]["level"] = "DEBUG"
-            logging_config["loggers"]["uvicorn.error"]["level"] = "DEBUG"
-            logging_config["loggers"]["uvicorn.access"]["level"] = "DEBUG"
-            logging_config["formatters"]["default"][
-                "fmt"
-            ] = "%(asctime)-15s %(name)-32s [%(levelname)-8s] (%(module)-17s) %(message)s"
-            logging_config["loggers"]["eodag"] = {
-                "handlers": ["default"],
-                "level": "DEBUG" if debug else "INFO",
-                "propagate": False,
+        uvicorn_fmt = "%(asctime)-15s %(name)-32s [%(levelname)-8s] %(message)s"
+        logging_config["filters"] = {"liveness": {"()": LivenessFilter}}
+        logging_config["formatters"]["access"]["fmt"] = uvicorn_fmt
+        logging_config["formatters"]["default"]["fmt"] = uvicorn_fmt
+        logging_config["loggers"]["uvicorn.access"]["filters"] = ["liveness"]
+
+        eodag_formatter = logging.Formatter(
+            "%(asctime)-15s %(name)-32s [%(levelname)-8s] (tid=%(thread)d) %(message)s"
+        )
+        logging.getLogger("eodag").handlers[0].setFormatter(eodag_formatter)
+
+        if ctx.obj["verbosity"] <= 1:
+            logging_config["handlers"]["null"] = {
+                "level": "DEBUG",
+                "class": "logging.NullHandler",
             }
+            logging_config["loggers"]["uvicorn"]["handlers"] = ["null"]
+            logging_config["loggers"]["uvicorn.error"]["handlers"] = ["null"]
+            logging_config["loggers"]["uvicorn.access"]["handlers"] = ["null"]
+        else:
+            log_level = "INFO" if ctx.obj["verbosity"] == 2 else "DEBUG"
+            logging_config["loggers"]["uvicorn"]["level"] = log_level
+            logging_config["loggers"]["uvicorn.error"]["level"] = log_level
+            logging_config["loggers"]["uvicorn.access"]["level"] = log_level
+
         uvicorn.run(
             "eodag.rest.server:app",
             host=bind_host,