eodag 3.8.1__py3-none-any.whl → 3.9.1__py3-none-any.whl

eodag/api/core.py

@@ -74,6 +74,7 @@ from eodag.utils import (
     string_to_jsonpath,
     uri_to_path,
 )
+from eodag.utils.dates import rfc3339_str_to_datetime
 from eodag.utils.env import is_env_var_true
 from eodag.utils.exceptions import (
     AuthenticationError,
@@ -84,7 +85,6 @@ from eodag.utils.exceptions import (
     UnsupportedProvider,
 )
 from eodag.utils.free_text_search import compile_free_text_query
-from eodag.utils.rest import rfc3339_str_to_datetime
 from eodag.utils.stac_reader import fetch_stac_items
 
 if TYPE_CHECKING:

eodag/api/product/drivers/generic.py

@@ -33,7 +33,11 @@ class GenericDriver(DatasetDriver):
         # data
         {
             "pattern": re.compile(
-                r"^(?:.*[/\\])?([^/\\]+)(\.jp2|\.tiff?|\.dat|\.nc|\.grib2?)$",
+                (
+                    r"^(?:.*[/\\])?([^/\\]+)"
+                    r"(\.jp2|\.tiff?|\.dat|\.nc|\.grib2?|"
+                    r"\.zarr|\.nat|\.covjson|\.parquet|\.zip|\.tar|\.gz)$"
+                ),
                 re.IGNORECASE,
             ),
             "roles": ["data"],

eodag/api/product/metadata_mapping.py

@@ -47,7 +47,6 @@ from eodag.utils import (
     dict_items_recursive_apply,
     format_string,
     get_geometry_from_various,
-    get_timestamp,
     items_recursive_apply,
     nested_pairs2dict,
     remove_str_array_quotes,
@@ -55,6 +54,7 @@ from eodag.utils import (
     string_to_jsonpath,
     update_nested_dict,
 )
+from eodag.utils.dates import get_timestamp
 from eodag.utils.exceptions import ValidationError
 
 if TYPE_CHECKING:
@@ -149,39 +149,38 @@ def format_metadata(search_param: str, *args: Any, **kwargs: Any) -> str:
     """Format a string of form ``{<field_name>#<conversion_function>}``
 
     The currently understood converters are:
-        - ``datetime_to_timestamp_milliseconds``: converts a utc date string to a timestamp in
-          milliseconds
-        - ``to_rounded_wkt``: simplify the WKT of a geometry
-        - ``to_bounds_lists``: convert to list(s) of bounds
-        - ``to_nwse_bounds``: convert to North,West,South,East bounds
-        - ``to_nwse_bounds_str``: convert to North,West,South,East bounds string with given separator
-        - ``to_geojson``: convert to a GeoJSON (via __geo_interface__ if exists)
+        - ``ceda_collection_name``: generate a CEDA collection name from a string
+        - ``csv_list``: convert to a comma separated list
+        - ``datetime_to_timestamp_milliseconds``: converts a utc date string to a timestamp in milliseconds
+        - ``dict_filter_and_sub``: filter dict items using jsonpath and then apply recursive_sub_str
+        - ``fake_l2a_title_from_l1c``: used to generate SAFE format metadata for data from AWS
+        - ``from_alternate``: update assets using given alternate
         - ``from_ewkt``: convert EWKT to shapely geometry / WKT in DEFAULT_PROJ
-        - ``to_ewkt``: convert to EWKT (Extended Well-Known text)
         - ``from_georss``: convert GeoRSS to shapely geometry / WKT in DEFAULT_PROJ
-        - ``csv_list``: convert to a comma separated list
-        - ``to_iso_utc_datetime_from_milliseconds``: convert a utc timestamp in given
-          milliseconds to a utc iso datetime
-        - ``to_iso_utc_datetime``: convert a UTC datetime string to ISO UTC datetime
-          string
-        - ``to_iso_date``: remove the time part of a iso datetime string
-        - ``remove_extension``: on a string that contains dots, only take the first
-          part of the list obtained by splitting the string on dots
+        - ``get_ecmwf_time``: get the time of a datetime string in the ECMWF format
         - ``get_group_name``: get the matching regex group name
+        - ``recursive_sub_str``: recursively substitue in the structure (e.g. dict) values matching a regex
+        - ``remove_extension``: on a string that contains dots, only take the first part of the list obtained by
+          splitting the string on dots
         - ``replace_str``: execute "string".replace(old, new)
-        - ``recursive_sub_str``: recursively substitue in the structure (e.g. dict)
-          values matching a regex
-        - ``slice_str``: slice a string (equivalent to s[start, end, step])
-        - ``to_lower``: Convert a string to lowercase
-        - ``to_upper``: Convert a string to uppercase
-        - ``fake_l2a_title_from_l1c``: used to generate SAFE format metadata for data from AWS
         - ``s2msil2a_title_to_aws_productinfo``: used to generate SAFE format metadata for data from AWS
+        - ``sanitize``: sanitize string
+        - ``slice_str``: slice a string (equivalent to s[start, end, step])
         - ``split_cop_dem_id``: get the bbox by splitting the product id
         - ``split_corine_id``: get the product type by splitting the product id
+        - ``to_bounds_lists``: convert to list(s) of bounds
         - ``to_datetime_dict``: convert a datetime string to a dictionary where values are either a string or a list
-        - ``get_ecmwf_time``: get the time of a datetime string in the ECMWF format
-        - ``sanitize``: sanitize string
-        - ``ceda_collection_name``: generate a CEDA collection name from a string
+        - ``to_ewkt``: convert to EWKT (Extended Well-Known text)
+        - ``to_geojson``: convert to a GeoJSON (via __geo_interface__ if exists)
+        - ``to_iso_date``: remove the time part of a iso datetime string
+        - ``to_iso_utc_datetime_from_milliseconds``: convert a utc timestamp in given milliseconds to a utc iso datetime
+        - ``to_iso_utc_datetime``: convert a UTC datetime string to ISO UTC datetime string
+        - ``to_lower``: Convert a string to lowercase
+        - ``to_nwse_bounds_str``: convert to North,West,South,East bounds string with given separator
+        - ``to_nwse_bounds``: convert to North,West,South,East bounds
+        - ``to_rounded_wkt``: simplify the WKT of a geometry
+        - ``to_title``: Convert a string to title case
+        - ``to_upper``: Convert a string to uppercase
 
     :param search_param: The string to be formatted
     :param args: (optional) Additional arguments to use in the formatting process
@@ -487,9 +486,9 @@ def format_metadata(search_param: str, *args: Any, **kwargs: Any) -> str:
             return {"lon": lon, "lat": lat}
 
         @staticmethod
-        def convert_csv_list(values_list: Any) -> Any:
+        def convert_csv_list(values_list: Any, separator=",") -> Any:
             if isinstance(values_list, list):
-                return ",".join([str(x) for x in values_list])
+                return separator.join([str(x) for x in values_list])
             else:
                 return values_list
 
@@ -529,6 +528,35 @@ def format_metadata(search_param: str, *args: Any, **kwargs: Any) -> str:
             old, new = ast.literal_eval(args)
             return re.sub(old, new, value)
 
+        @staticmethod
+        def convert_replace_str_tuple(value: Any, args: str) -> str:
+            """
+            Apply multiple replacements on a string.
+            args should be a string representing a list/tuple of (old, new) pairs.
+            Example: '(("old1", "new1"), ("old2", "new2"))'
+            """
+            if isinstance(value, dict):
+                value = MetadataFormatter.convert_to_geojson(value)
+            elif not isinstance(value, str):
+                raise TypeError(
+                    f"convert_replace_str_tuple expects a string or a dict (apply to_geojson). "
+                    f"Got {type(value)}: {value}"
+                )
+
+            # args sera une chaîne représentant une liste/tuple de tuples
+            replacements = ast.literal_eval(args)
+
+            if not isinstance(replacements, (list, tuple)):
+                raise TypeError(
+                    f"convert_replace_str_tuple expects a list/tuple of (old,new) pairs. "
+                    f"Got {type(replacements)}: {replacements}"
+                )
+
+            for old, new in replacements:
+                value = re.sub(old, new, value)
+
+            return value
+
         @staticmethod
         def convert_ceda_collection_name(value: str) -> str:
             data_regex = re.compile(r"/data/(?P<name>.+?)/?$")
@@ -580,6 +608,45 @@ def format_metadata(search_param: str, *args: Any, **kwargs: Any) -> str:
                 result[key] = match.value
             return result
 
+        @staticmethod
+        def convert_dict_filter_and_sub(
+            input_dict: dict[Any, Any], args: str
+        ) -> Union[dict[Any, Any], list[Any]]:
+            """Fitlers dict items using jsonpath and then apply recursive_sub_str"""
+            jsonpath_filter_str, old, new = ast.literal_eval(args)
+            filtered = MetadataFormatter.convert_dict_filter(
+                input_dict, jsonpath_filter_str
+            )
+            args_str = f"('{old}', '{new}')"
+            return MetadataFormatter.convert_recursive_sub_str(filtered, args_str)
+
+        @staticmethod
+        def convert_from_alternate(
+            input_obj: dict[str, Any], value: str
+        ) -> dict[str, Any]:
+            """
+            Update assets using given alternate.
+            """
+            result: dict[str, Any] = {}
+            for k, v in input_obj.items():
+                if not isinstance(v, dict):
+                    continue
+
+                alt_dict = deepcopy(v).get("alternate")
+                if not isinstance(alt_dict, dict):
+                    continue
+
+                value_entry = alt_dict.pop(value, None)
+                if not isinstance(value_entry, dict):
+                    continue
+
+                result[k] = v | value_entry | {"alternate": alt_dict}
+
+                if len(result[k]["alternate"]) == 0:
+                    del result[k]["alternate"]
+
+            return result
+
         @staticmethod
         def convert_slice_str(string: str, args: str) -> str:
             cmin, cmax, cstep = [
@@ -591,6 +658,8 @@ def format_metadata(search_param: str, *args: Any, **kwargs: Any) -> str:
         @staticmethod
         def convert_to_lower(string: str) -> str:
             """Convert a string to lowercase."""
+            if string == NOT_AVAILABLE:
+                return string
             return string.lower()
 
         @staticmethod
@@ -598,6 +667,13 @@ def format_metadata(search_param: str, *args: Any, **kwargs: Any) -> str:
             """Convert a string to uppercase."""
             return string.upper()
 
+        @staticmethod
+        def convert_to_title(string: str) -> str:
+            """Convert a string to title case."""
+            if string == NOT_AVAILABLE:
+                return string
+            return string.title()
+
         @staticmethod
         def convert_fake_l2a_title_from_l1c(string: str) -> str:
             id_regex = re.compile(
@@ -1362,17 +1438,30 @@ def format_query_params(
         error_context,
     )
 
-    for eodag_search_key, provider_search_key in queryables.items():
+    for eodag_search_key, provider_search_param in queryables.items():
         user_input = query_dict[eodag_search_key]
 
-        if COMPLEX_QS_REGEX.match(provider_search_key):
-            parts = provider_search_key.split("=")
+        if provider_search_param == user_input:
+            # means the mapping is to be passed as is, in which case we
+            # readily register it
+            if (
+                eodag_search_key in query_params
+                and isinstance(query_params[eodag_search_key], dict)
+                and isinstance(user_input, dict)
+            ):
+                query_params[eodag_search_key].update(user_input)
+            else:
+                query_params[eodag_search_key] = user_input
+            continue
+
+        if COMPLEX_QS_REGEX.match(provider_search_param):
+            parts = provider_search_param.split("=")
             if len(parts) == 1:
                 formatted_query_param = format_metadata(
-                    provider_search_key, product_type, **query_dict
+                    provider_search_param, product_type, **query_dict
                 )
                 formatted_query_param = formatted_query_param.replace("'", '"')
-                if "{{" in provider_search_key:
+                if "{{" in provider_search_param:
                     # retrieve values from hashes where keys are given in the param
                     if "}[" in formatted_query_param:
                         formatted_query_param = _resolve_hashes(formatted_query_param)
@@ -1396,7 +1485,7 @@ def format_query_params(
                     provider_value, product_type, **query_dict
                 )
         else:
-            query_params[provider_search_key] = user_input
+            query_params[provider_search_param] = user_input
     # Now get all the literal search params (i.e params to be passed "as is"
     # in the search request)
     # ignore additional_params if it isn't a dictionary
@@ -1527,7 +1616,15 @@ def _get_queryables(
                     config.discover_metadata.get("metadata_pattern", "")
                 )
                 search_param_cfg = config.discover_metadata.get("search_param", "")
-                if pattern.match(eodag_search_key) and isinstance(
+                search_param_unparsed_cfg = config.discover_metadata.get(
+                    "search_param_unparsed", []
+                )
+                if (
+                    search_param_unparsed_cfg
+                    and eodag_search_key in search_param_unparsed_cfg
+                ):
+                    queryables[eodag_search_key] = user_input
+                elif pattern.match(eodag_search_key) and isinstance(
                     search_param_cfg, str
                 ):
                     search_param = search_param_cfg.format(metadata=eodag_search_key)

eodag/cli.py

@@ -42,13 +42,14 @@ Commands:
 
 from __future__ import annotations
 
+import functools
 import json
 import os
 import shutil
 import sys
 import textwrap
 from importlib.metadata import metadata
-from typing import TYPE_CHECKING, Any, Mapping
+from typing import TYPE_CHECKING, Any, Callable, Mapping, Optional
 from urllib.parse import parse_qs
 
 import click
@@ -118,6 +119,22 @@ class MutuallyExclusiveOption(click.Option):
         return super(MutuallyExclusiveOption, self).handle_parse_result(ctx, opts, args)
 
 
+def _deprecated_cli(message: str, version: Optional[str] = None) -> Callable[..., Any]:
+    """Decorator to mark a CLI command as deprecated and print a bold yellow warning."""
+    version_msg = f" -- Deprecated since v{version}" if version else ""
+
+    def decorator(func: Callable[..., Any]) -> Callable[..., Any]:
+        @functools.wraps(func)
+        def wrapper(*args: Any, **kwargs: Any) -> Any:
+            full_message = f"DEPRECATED: {message}{version_msg}"
+            click.echo(click.style(full_message, fg="yellow", bold=True), err=True)
+            return func(*args, **kwargs)
+
+        return wrapper
+
+    return decorator
+
+
 @click.group(chain=True)
 @click.option(
     "-v",
@@ -631,9 +648,17 @@ def download(ctx: Context, **kwargs: Any) -> None:
 
 
 @eodag.command(
-    help="Start eodag HTTP server\n\n"
-    "Set EODAG_CORS_ALLOWED_ORIGINS environment variable to configure Cross-Origin Resource Sharing allowed origins as "
-    "comma-separated URLs (e.g. 'http://somewhere,htttp://somewhere.else')."
+    help="(deprecated) Start eodag HTTP server\n\n"
+    + (
+        click.style(
+            "Running a web server from the CLI is deprecated and will be removed in a future version.\n"
+            "This feature has been moved to its own repository: https://github.com/CS-SI/stac-fastapi-eodag\n\n",
+            fg="yellow",
+            bold=True,
+        )
+        + "Set EODAG_CORS_ALLOWED_ORIGINS environment variable to configure Cross-Origin Resource Sharing allowed "
+        "origins as comma-separated URLs (e.g. 'http://somewhere,http://somewhere.else')."
+    )
 )
 @click.option(
     "-f",
@@ -676,6 +701,13 @@ def download(ctx: Context, **kwargs: Any) -> None:
     help="Run in debug mode (for development purpose)",
 )
 @click.pass_context
+@_deprecated_cli(
+    message=(
+        "Running a web server from the CLI is deprecated and will be removed in a future version. "
+        "This feature has been moved to its own repository: https://github.com/CS-SI/stac-fastapi-eodag"
+    ),
+    version="3.9.0",
+)
 def serve_rest(
     ctx: Context,
     daemon: bool,

eodag/config.py

@@ -272,6 +272,8 @@ class PluginConfig(yaml.YAMLObject):
         search_param: str | dict[str, Any]
         #: Path to the metadata in search result
         metadata_path: str
+        #: list search parameters to send as is to the provider
+        search_param_unparsed: list[str]
         #: Whether an error must be raised when using a search parameter which is not queryable or not
         raise_mtd_discovery_error: bool
 
@@ -543,8 +545,6 @@ class PluginConfig(yaml.YAMLObject):
     #: :class:`~eodag.plugins.download.s3rest.S3RestDownload`
     #: At which level of the path part of the url the bucket can be found
     bucket_path_level: int
-    #: :class:`~eodag.plugins.download.aws.AwsDownload` Whether download is done from a requester-pays bucket or not
-    requester_pays: bool
     #: :class:`~eodag.plugins.download.aws.AwsDownload` S3 endpoint
     s3_endpoint: str
 
@@ -571,6 +571,9 @@ class PluginConfig(yaml.YAMLObject):
     #: :class:`~eodag.plugins.authentication.base.Authentication` Part of the search or download plugin configuration
     #: that needs authentication
     matching_conf: dict[str, Any]
+    #: :class:`~eodag.plugins.authentication.aws_auth.AwsAuth`
+    #: Whether download is done from a requester-pays bucket or not
+    requester_pays: bool
     #: :class:`~eodag.plugins.authentication.openid_connect.OIDCRefreshTokenBase`
     #: How the token should be used in the request
     token_provision: str

eodag/plugins/apis/ecmwf.py

@@ -46,6 +46,7 @@ from eodag.utils.logging import get_logging_verbose
 if TYPE_CHECKING:
     from typing import Any, Optional, Union
 
+    from mypy_boto3_s3 import S3ServiceResource
     from requests.auth import AuthBase
 
     from eodag.api.product import EOProduct
@@ -55,6 +56,7 @@ if TYPE_CHECKING:
     from eodag.types.download_args import DownloadConf
     from eodag.utils import DownloadedCallback, ProgressCallback, Unpack
 
+
 logger = logging.getLogger("eodag.apis.ecmwf")
 
 ECMWF_MARS_KNOWN_FORMATS = {"grib": "grib", "netcdf": "nc"}
@@ -171,7 +173,7 @@ class EcmwfApi(Api, ECMWFSearch):
     def download(
         self,
         product: EOProduct,
-        auth: Optional[Union[AuthBase, S3SessionKwargs]] = None,
+        auth: Optional[Union[AuthBase, S3SessionKwargs, S3ServiceResource]] = None,
         progress_callback: Optional[ProgressCallback] = None,
         wait: float = DEFAULT_DOWNLOAD_WAIT,
         timeout: float = DEFAULT_DOWNLOAD_TIMEOUT,

eodag/plugins/apis/usgs.py

@@ -57,6 +57,7 @@ from eodag.utils.exceptions import (
 )
 
 if TYPE_CHECKING:
+    from mypy_boto3_s3 import S3ServiceResource
     from requests.auth import AuthBase
 
     from eodag.api.search_result import SearchResult
@@ -296,7 +297,7 @@ class UsgsApi(Api):
     def download(
         self,
         product: EOProduct,
-        auth: Optional[Union[AuthBase, S3SessionKwargs]] = None,
+        auth: Optional[Union[AuthBase, S3SessionKwargs, S3ServiceResource]] = None,
         progress_callback: Optional[ProgressCallback] = None,
         wait: float = DEFAULT_DOWNLOAD_WAIT,
         timeout: float = DEFAULT_DOWNLOAD_TIMEOUT,