eodag 3.0.1__py3-none-any.whl → 3.1.0__py3-none-any.whl

eodag/utils/__init__.py

@@ -54,14 +54,10 @@ from typing import (
     TYPE_CHECKING,
     Any,
     Callable,
-    Dict,
     Iterable,
     Iterator,
-    List,
     Mapping,
     Optional,
-    Tuple,
-    Type,
     Union,
     cast,
 )
@@ -84,6 +80,7 @@ if sys.version_info >= (3, 12):
 else:
     from typing_extensions import Unpack  # noqa
 
+
 import click
 import orjson
 import shapefile
@@ -118,6 +115,7 @@ eodag_version = metadata("eodag")["Version"]
 USER_AGENT = {"User-Agent": f"eodag/{eodag_version}"}
 
 HTTP_REQ_TIMEOUT = 5  # in seconds
+DEFAULT_SEARCH_TIMEOUT = 20  # in seconds
 DEFAULT_STREAM_REQUESTS_TIMEOUT = 60  # in seconds
 
 REQ_RETRY_TOTAL = 3
@@ -125,8 +123,8 @@ REQ_RETRY_BACKOFF_FACTOR = 2
 REQ_RETRY_STATUS_FORCELIST = [401, 429, 500, 502, 503, 504]
 
 # default wait times in minutes
-DEFAULT_DOWNLOAD_WAIT = 2  # in minutes
-DEFAULT_DOWNLOAD_TIMEOUT = 20  # in minutes
+DEFAULT_DOWNLOAD_WAIT = 0.2  # in minutes
+DEFAULT_DOWNLOAD_TIMEOUT = 10  # in minutes
 
 JSONPATH_MATCH = re.compile(r"^[\{\(]*\$(\..*)*$")
 WORKABLE_JSONPATH_MATCH = re.compile(r"^\$(\.[a-zA-Z0-9-_:\.\[\]\"\(\)=\?\*]+)*$")
@@ -142,6 +140,13 @@ DEFAULT_MAX_ITEMS_PER_PAGE = 50
 # default product-types start date
 DEFAULT_MISSION_START_DATE = "2015-01-01T00:00:00Z"
 
+# update missing mimetypes
+mimetypes.add_type("text/xml", ".xsd")
+mimetypes.add_type("application/x-grib", ".grib")
+mimetypes.add_type("application/x-grib2", ".grib2")
+# jp2 is missing on windows
+mimetypes.add_type("image/jp2", ".jp2")
+
 
 def _deprecated(reason: str = "", version: Optional[str] = None) -> Callable[..., Any]:
     """Simple decorator to mark functions/methods/classes as deprecated.
@@ -314,7 +319,7 @@ def path_to_uri(path: str) -> str:
     return Path(path).as_uri()
 
 
-def mutate_dict_in_place(func: Callable[[Any], Any], mapping: Dict[Any, Any]) -> None:
+def mutate_dict_in_place(func: Callable[[Any], Any], mapping: dict[Any, Any]) -> None:
     """Apply func to values of mapping.
 
     The mapping object's values are modified in-place. The function is recursive,
@@ -332,7 +337,7 @@ def mutate_dict_in_place(func: Callable[[Any], Any], mapping: Dict[Any, Any]) ->
             mapping[key] = func(value)
 
 
-def merge_mappings(mapping1: Dict[Any, Any], mapping2: Dict[Any, Any]) -> None:
+def merge_mappings(mapping1: dict[Any, Any], mapping2: dict[Any, Any]) -> None:
     """Merge two mappings with string keys, values from ``mapping2`` overriding values
     from ``mapping1``.
 
@@ -430,6 +435,33 @@ def datetime_range(start: dt, end: dt) -> Iterator[dt]:
         yield start + datetime.timedelta(days=nday)
 
 
+def is_range_in_range(valid_range: str, check_range: str) -> bool:
+    """Check if the check_range is completely within the valid_range.
+
+    This function checks if both the start and end dates of the check_range
+    are within the start and end dates of the valid_range.
+
+    :param valid_range: The valid date range in the format 'YYYY-MM-DD/YYYY-MM-DD'.
+    :param check_range: The date range to check in the format 'YYYY-MM-DD/YYYY-MM-DD'.
+    :returns: True if check_range is within valid_range, otherwise False.
+    """
+    if "/" not in valid_range or "/" not in check_range:
+        return False
+
+    # Split the date ranges into start and end dates
+    start_valid, end_valid = valid_range.split("/")
+    start_check, end_check = check_range.split("/")
+
+    # Convert the strings to datetime objects using fromisoformat
+    start_valid_dt = datetime.datetime.fromisoformat(start_valid)
+    end_valid_dt = datetime.datetime.fromisoformat(end_valid)
+    start_check_dt = datetime.datetime.fromisoformat(start_check)
+    end_check_dt = datetime.datetime.fromisoformat(end_check)
+
+    # Check if check_range is within valid_range
+    return start_valid_dt <= start_check_dt and end_valid_dt >= end_check_dt
+
+
 class DownloadedCallback:
     """Example class for callback after each download in :meth:`~eodag.api.core.EODataAccessGateway.download_all`"""
 
@@ -570,9 +602,49 @@ def rename_subfolder(dirpath: str, name: str) -> None:
     )
 
 
+def rename_with_version(file_path: str, suffix: str = "old") -> str:
+    """
+    Renames a file by appending and incrementing a version number if a conflict exists.
+
+    :param file_path: full path of the file to rename
+    :param suffix: suffix preceding version number in case of name conflict
+    :returns: new file path with the version appended or incremented
+
+    Example:
+
+    >>> import tempfile
+    >>> from pathlib import Path
+    >>> with tempfile.TemporaryDirectory() as tmpdir:
+    ...     file_path = (Path(tmpdir) / "foo.txt")
+    ...     file_path.touch()
+    ...     (Path(tmpdir) / "foo_old1.txt").touch()
+    ...     expected = str(Path(tmpdir) / "foo_old2.txt")
+    ...     assert expected == rename_with_version(str(file_path))
+
+    """
+    if not os.path.isfile(file_path):
+        raise FileNotFoundError(f"The file '{file_path}' does not exist.")
+
+    dir_path, file_name = os.path.split(file_path)
+    file_base, file_ext = os.path.splitext(file_name)
+
+    new_file_path = file_path
+
+    # loop and iterate on conflicting existing files
+    version = 0
+    while os.path.exists(new_file_path):
+        version += 1
+        new_file_name = f"{file_base}_{suffix}{version}{file_ext}"
+        new_file_path = os.path.join(dir_path, new_file_name)
+
+    # Rename the file
+    os.rename(file_path, new_file_path)
+    return new_file_path
+
+
 def format_dict_items(
-    config_dict: Dict[str, Any], **format_variables: Any
-) -> Dict[Any, Any]:
+    config_dict: dict[str, Any], **format_variables: Any
+) -> dict[Any, Any]:
     r"""Recursively apply :meth:`str.format` to ``**format_variables`` on ``config_dict`` values
 
     >>> format_dict_items(
@@ -589,8 +661,8 @@ def format_dict_items(
 
 
 def jsonpath_parse_dict_items(
-    jsonpath_dict: Dict[str, Any], values_dict: Dict[str, Any]
-) -> Dict[Any, Any]:
+    jsonpath_dict: dict[str, Any], values_dict: dict[str, Any]
+) -> dict[Any, Any]:
     """Recursively parse :class:`jsonpath_ng.JSONPath` elements in dict
 
     >>> import jsonpath_ng.ext as jsonpath
@@ -608,12 +680,12 @@ def jsonpath_parse_dict_items(
 
 
 def update_nested_dict(
-    old_dict: Dict[Any, Any],
-    new_dict: Dict[Any, Any],
+    old_dict: dict[Any, Any],
+    new_dict: dict[Any, Any],
     extend_list_values: bool = False,
     allow_empty_values: bool = False,
     allow_extend_duplicates: bool = True,
-) -> Dict[Any, Any]:
+) -> dict[Any, Any]:
     """Update recursively ``old_dict`` items with ``new_dict`` ones
 
     >>> update_nested_dict(
@@ -693,10 +765,10 @@ def update_nested_dict(
 
 
 def items_recursive_apply(
-    input_obj: Union[Dict[Any, Any], List[Any]],
+    input_obj: Union[dict[Any, Any], list[Any]],
     apply_method: Callable[..., Any],
     **apply_method_parameters: Any,
-) -> Union[Dict[Any, Any], List[Any]]:
+) -> Union[dict[Any, Any], list[Any]]:
     """Recursive apply method to items contained in input object (dict or list)
 
     >>> items_recursive_apply(
@@ -734,10 +806,10 @@ def items_recursive_apply(
 
 
 def dict_items_recursive_apply(
-    config_dict: Dict[Any, Any],
+    config_dict: dict[Any, Any],
     apply_method: Callable[..., Any],
     **apply_method_parameters: Any,
-) -> Dict[Any, Any]:
+) -> dict[Any, Any]:
     """Recursive apply method to dict elements
 
     >>> dict_items_recursive_apply(
@@ -751,7 +823,7 @@ def dict_items_recursive_apply(
     :param apply_method_parameters: Optional parameters passed to the method
     :returns: Updated dict
     """
-    result_dict: Dict[Any, Any] = deepcopy(config_dict)
+    result_dict: dict[Any, Any] = deepcopy(config_dict)
     for dict_k, dict_v in result_dict.items():
         if isinstance(dict_v, dict):
             result_dict[dict_k] = dict_items_recursive_apply(
@@ -759,7 +831,7 @@ def dict_items_recursive_apply(
             )
         elif any(isinstance(dict_v, t) for t in (list, tuple)):
             result_dict[dict_k] = list_items_recursive_apply(
-                dict_v, apply_method, **apply_method_parameters
+                list(dict_v), apply_method, **apply_method_parameters
             )
         else:
             result_dict[dict_k] = apply_method(
@@ -770,10 +842,10 @@ def dict_items_recursive_apply(
 
 
 def list_items_recursive_apply(
-    config_list: List[Any],
+    config_list: list[Any],
     apply_method: Callable[..., Any],
     **apply_method_parameters: Any,
-) -> List[Any]:
+) -> list[Any]:
     """Recursive apply method to list elements
 
     >>> list_items_recursive_apply(
@@ -806,8 +878,8 @@ def list_items_recursive_apply(
 
 
 def items_recursive_sort(
-    input_obj: Union[List[Any], Dict[Any, Any]],
-) -> Union[List[Any], Dict[Any, Any]]:
+    input_obj: Union[list[Any], dict[Any, Any]],
+) -> Union[list[Any], dict[Any, Any]]:
     """Recursive sort dict items contained in input object (dict or list)
 
     >>> items_recursive_sort(
@@ -831,7 +903,7 @@ def items_recursive_sort(
         return input_obj
 
 
-def dict_items_recursive_sort(config_dict: Dict[Any, Any]) -> Dict[Any, Any]:
+def dict_items_recursive_sort(config_dict: dict[Any, Any]) -> dict[Any, Any]:
     """Recursive sort dict elements
 
     >>> dict_items_recursive_sort(
@@ -842,7 +914,7 @@ def dict_items_recursive_sort(config_dict: Dict[Any, Any]) -> Dict[Any, Any]:
     :param config_dict: Input nested dictionary
     :returns: Updated dict
     """
-    result_dict: Dict[Any, Any] = deepcopy(config_dict)
+    result_dict: dict[Any, Any] = deepcopy(config_dict)
     for dict_k, dict_v in result_dict.items():
         if isinstance(dict_v, dict):
             result_dict[dict_k] = dict_items_recursive_sort(dict_v)
@@ -854,7 +926,7 @@ def dict_items_recursive_sort(config_dict: Dict[Any, Any]) -> Dict[Any, Any]:
     return dict(sorted(result_dict.items()))
 
 
-def list_items_recursive_sort(config_list: List[Any]) -> List[Any]:
+def list_items_recursive_sort(config_list: list[Any]) -> list[Any]:
     """Recursive sort dicts in list elements
 
     >>> list_items_recursive_sort(["b", {2: 0, 0: 1, 1: 2}])
@@ -863,7 +935,7 @@ def list_items_recursive_sort(config_list: List[Any]) -> List[Any]:
     :param config_list: Input list containing nested lists/dicts
     :returns: Updated list
     """
-    result_list: List[Any] = deepcopy(config_list)
+    result_list: list[Any] = deepcopy(config_list)
     for list_idx, list_v in enumerate(result_list):
         if isinstance(list_v, dict):
             result_list[list_idx] = dict_items_recursive_sort(list_v)
@@ -976,7 +1048,7 @@ def format_string(key: str, str_to_format: Any, **format_variables: Any) -> Any:
         # defaultdict usage will return "" for missing keys in format_args
         try:
             result = str_to_format.format_map(defaultdict(str, **format_variables))
-        except TypeError as e:
+        except (ValueError, TypeError) as e:
             raise MisconfiguredError(
                 f"Unable to format str={str_to_format} using {str(format_variables)}: {str(e)}"
             )
@@ -989,7 +1061,7 @@ def format_string(key: str, str_to_format: Any, **format_variables: Any) -> Any:
 
 
 def parse_jsonpath(
-    key: str, jsonpath_obj: Union[str, jsonpath.Child], **values_dict: Dict[str, Any]
+    key: str, jsonpath_obj: Union[str, jsonpath.Child], **values_dict: dict[str, Any]
 ) -> Optional[str]:
     """Parse jsonpah in ``jsonpath_obj`` using ``values_dict``
 
@@ -1009,7 +1081,7 @@ def parse_jsonpath(
         return jsonpath_obj
 
 
-def nested_pairs2dict(pairs: Union[List[Any], Any]) -> Union[Any, Dict[Any, Any]]:
+def nested_pairs2dict(pairs: Union[list[Any], Any]) -> Union[Any, dict[Any, Any]]:
     """Create a dict using nested pairs
 
     >>> nested_pairs2dict([["foo", [["bar", "baz"]]]])
@@ -1031,7 +1103,7 @@ def nested_pairs2dict(pairs: Union[List[Any], Any]) -> Union[Any, Dict[Any, Any]
 
 
 def get_geometry_from_various(
-    locations_config: List[Dict[str, Any]] = [], **query_args: Any
+    locations_config: list[dict[str, Any]] = [], **query_args: Any
 ) -> BaseGeometry:
     """Creates a ``shapely.geometry`` using given query kwargs arguments
 
@@ -1120,7 +1192,7 @@ def get_geometry_from_various(
 class MockResponse:
     """Fake requests response"""
 
-    def __init__(self, json_data: Any, status_code: int) -> None:
+    def __init__(self, json_data: Any = None, status_code: int = 200) -> None:
         self.json_data = json_data
         self.status_code = status_code
         self.content = json_data
@@ -1129,10 +1201,21 @@ class MockResponse:
         """Return json data"""
         return self.json_data
 
+    def __iter__(self):
+        yield self
+
+    def __enter__(self):
+        return self
+
+    def __exit__(self, exc_type, exc_val, exc_tb):
+        pass
+
     def raise_for_status(self) -> None:
         """raises an exception when the status is not ok"""
         if self.status_code != 200:
-            raise HTTPError(response=Response())
+            response = Response()
+            response.status_code = self.status_code
+            raise HTTPError(response=response)
 
 
 def md5sum(file_path: str) -> str:
@@ -1196,7 +1279,7 @@ def _mutable_cached_yaml_load(config_path: str) -> Any:
         return yaml.load(fh, Loader=yaml.SafeLoader)
 
 
-def cached_yaml_load(config_path: str) -> Dict[str, Any]:
+def cached_yaml_load(config_path: str) -> dict[str, Any]:
     """Cached :func:`yaml.load`
 
     :param config_path: path to the yaml configuration file
@@ -1206,12 +1289,12 @@ def cached_yaml_load(config_path: str) -> Dict[str, Any]:
 
 
 @functools.lru_cache()
-def _mutable_cached_yaml_load_all(config_path: str) -> List[Any]:
+def _mutable_cached_yaml_load_all(config_path: str) -> list[Any]:
     with open(config_path, "r") as fh:
         return list(yaml.load_all(fh, Loader=yaml.Loader))
 
 
-def cached_yaml_load_all(config_path: str) -> List[Any]:
+def cached_yaml_load_all(config_path: str) -> list[Any]:
     """Cached :func:`yaml.load_all`
 
     Load all configurations stored in the configuration file as separated yaml documents
@@ -1224,7 +1307,7 @@ def cached_yaml_load_all(config_path: str) -> List[Any]:
 
 def get_bucket_name_and_prefix(
     url: str, bucket_path_level: Optional[int] = None
-) -> Tuple[Optional[str], Optional[str]]:
+) -> tuple[Optional[str], Optional[str]]:
     """Extract bucket name and prefix from URL
 
     :param url: (optional) URL to use as product.location
@@ -1237,7 +1320,9 @@ def get_bucket_name_and_prefix(
     subdomain = netloc.split(".")[0]
     path = path.strip("/")
 
-    if scheme and bucket_path_level is None:
+    if "/" in path and scheme and subdomain == "s3" and bucket_path_level is None:
+        bucket, prefix = path.split("/", 1)
+    elif scheme and bucket_path_level is None:
         bucket = subdomain
         prefix = path
     elif not scheme and bucket_path_level is None:
@@ -1283,10 +1368,10 @@ def deepcopy(sth: Any) -> Any:
     :param sth: Object to copy
     :returns: Copied object
     """
-    _dispatcher: Dict[Type[Any], Callable[..., Any]] = {}
+    _dispatcher: dict[type[Any], Callable[..., Any]] = {}
 
     def _copy_list(
-        input_list: List[Any], dispatch: Dict[Type[Any], Callable[..., Any]]
+        input_list: list[Any], dispatch: dict[type[Any], Callable[..., Any]]
     ):
         ret = input_list.copy()
         for idx, item in enumerate(ret):
@@ -1296,7 +1381,7 @@ def deepcopy(sth: Any) -> Any:
         return ret
 
     def _copy_dict(
-        input_dict: Dict[Any, Any], dispatch: Dict[Type[Any], Callable[..., Any]]
+        input_dict: dict[Any, Any], dispatch: dict[type[Any], Callable[..., Any]]
     ):
         ret = input_dict.copy()
         for key, value in ret.items():
@@ -1373,17 +1458,34 @@ class StreamResponse:
 
 
 def guess_file_type(file: str) -> Optional[str]:
-    """guess the mime type of a file or URL based on its extension"""
-    mimetypes.add_type("text/xml", ".xsd")
-    mimetypes.add_type("application/x-grib", ".grib")
+    """Guess the mime type of a file or URL based on its extension,
+    using eodag extended mimetypes definition
+
+    >>> guess_file_type('foo.tiff')
+    'image/tiff'
+    >>> guess_file_type('foo.grib')
+    'application/x-grib'
+
+    :param file: file url or path
+    :returns: guessed mime type
+    """
     mime_type, _ = mimetypes.guess_type(file, False)
+    if mime_type == "text/xml":
+        return "application/xml"
     return mime_type
 
 
 def guess_extension(type: str) -> Optional[str]:
-    """guess extension from mime type"""
-    mimetypes.add_type("text/xml", ".xsd")
-    mimetypes.add_type("application/x-grib", ".grib")
+    """Guess extension from mime type, using eodag extended mimetypes definition
+
+    >>> guess_extension('image/tiff')
+    '.tiff'
+    >>> guess_extension('application/x-grib')
+    '.grib'
+
+    :param type: mime type
+    :returns: guessed file extension
+    """
     return mimetypes.guess_extension(type, strict=False)
 
 
@@ -1404,7 +1506,7 @@ def get_ssl_context(ssl_verify: bool) -> ssl.SSLContext:
     return ctx
 
 
-def sort_dict(input_dict: Dict[str, Any]) -> Dict[str, Any]:
+def sort_dict(input_dict: dict[str, Any]) -> dict[str, Any]:
     """
     Recursively sorts a dict by keys.
 
@@ -1420,7 +1522,7 @@ def sort_dict(input_dict: Dict[str, Any]) -> Dict[str, Any]:
     }
 
 
-def dict_md5sum(input_dict: Dict[str, Any]) -> str:
+def dict_md5sum(input_dict: dict[str, Any]) -> str:
     """
     Hash nested dictionary
 

eodag/utils/exceptions.py

@@ -20,23 +20,15 @@ from __future__ import annotations
 from typing import TYPE_CHECKING, Annotated
 
 if TYPE_CHECKING:
-    from typing import Optional, Set
+    from typing import Optional
 
-    from typing_extensions import Doc
+    from typing_extensions import Doc, Self
 
 
 class EodagError(Exception):
     """General EODAG error"""
 
 
-class ValidationError(EodagError):
-    """Error validating data"""
-
-    def __init__(self, message: str, parameters: Set[str] = set()) -> None:
-        self.message = message
-        self.parameters = parameters
-
-
 class PluginNotFoundError(EodagError):
     """Error when looking for a plugin class that was not defined"""
 
@@ -74,14 +66,19 @@ class AuthenticationError(EodagError):
     authenticating a user"""
 
 
-class DownloadError(EodagError):
-    """An error indicating something wrong with the download process"""
-
-
 class NotAvailableError(EodagError):
     """An error indicating that the product is not available for download"""
 
 
+class NoMatchingProductType(EodagError):
+    """An error indicating that eodag was unable to derive a product type from a set
+    of search parameters"""
+
+
+class STACOpenerError(EodagError):
+    """An error indicating that a STAC file could not be opened"""
+
+
 class RequestError(EodagError):
     """An error indicating that a request has failed. Usually eodag functions
     and methods should catch and skip this"""
@@ -89,7 +86,7 @@ class RequestError(EodagError):
     status_code: Annotated[Optional[int], Doc("HTTP status code")] = None
 
     @classmethod
-    def from_error(cls, error: Exception, msg: Optional[str] = None):
+    def from_error(cls, error: Exception, msg: Optional[str] = None) -> Self:
         """Generate a RequestError from an Exception"""
         status_code = getattr(error, "code", None)
         text = getattr(error, "msg", None)
@@ -99,7 +96,7 @@ class RequestError(EodagError):
         # have a status code other than 200
         if response is not None:
             status_code = response.status_code
-            text = response.text
+            text = " ".join([text or "", response.text])
 
         text = text or str(error)
 
@@ -108,13 +105,23 @@ class RequestError(EodagError):
         return e
 
 
-class NoMatchingProductType(EodagError):
-    """An error indicating that eodag was unable to derive a product type from a set
-    of search parameters"""
+class ValidationError(RequestError):
+    """Error validating data"""
 
+    def __init__(self, message: str, parameters: set[str] = set()) -> None:
+        self.message = message
+        self.parameters = parameters
 
-class STACOpenerError(EodagError):
-    """An error indicating that a STAC file could not be opened"""
+    @classmethod
+    def from_error(cls, error: Exception, msg: Optional[str] = None) -> Self:
+        """Override parent from_error to handle ValidationError specificities."""
+        setattr(error, "msg", msg)
+        validation_error = super().from_error(error)
+        return validation_error
+
+
+class DownloadError(RequestError):
+    """An error indicating something wrong with the download process"""
 
 
 class TimeOutError(RequestError):

eodag/utils/import_system.py

@@ -21,14 +21,14 @@ import importlib
 import pkgutil
 from contextlib import contextmanager
 from functools import partial
-from typing import TYPE_CHECKING, Any, Generator, Tuple
+from typing import TYPE_CHECKING, Any, Generator
 
 if TYPE_CHECKING:
     from types import ModuleType
 
 
 def import_all_modules(
-    base_package: ModuleType, depth: int = 1, exclude: Tuple[str, ...] = ()
+    base_package: ModuleType, depth: int = 1, exclude: tuple[str, ...] = ()
 ) -> None:
     """Import all modules in base_package, including modules in the sub-packages up to `depth` and excluding modules in
     `exclude`.