eodag 2.12.0__py3-none-any.whl → 3.0.0b1__py3-none-any.whl

eodag/config.py

@@ -27,6 +27,7 @@ from typing import (
     ItemsView,
     Iterator,
     List,
+    Literal,
     Optional,
     Tuple,
     TypedDict,
@@ -40,13 +41,16 @@ import requests
 import yaml
 import yaml.constructor
 import yaml.parser
+from annotated_types import Gt
 from jsonpath_ng import JSONPath
 from pkg_resources import resource_filename
 from requests.auth import AuthBase
+from typing_extensions import Doc
 
 from eodag.utils import (
     HTTP_REQ_TIMEOUT,
     USER_AGENT,
+    Annotated,
     cached_yaml_load,
     cached_yaml_load_all,
     cast_scalar_value,
@@ -125,7 +129,11 @@ class ProviderConfig(yaml.YAMLObject):
     """
 
     name: str
+    group: str
     priority: int = 0  # Set default priority to 0
+    roles: List[str]
+    description: str
+    url: str
     api: PluginConfig
     search: PluginConfig
     products: Dict[str, Any]
@@ -138,7 +146,7 @@ class ProviderConfig(yaml.YAMLObject):
     yaml_tag = "!provider"
 
     @classmethod
-    def from_yaml(cls, loader: yaml.Loader, node: Any) -> ProviderConfig:
+    def from_yaml(cls, loader: yaml.Loader, node: Any) -> Iterator[ProviderConfig]:
         """Build a :class:`~eodag.config.ProviderConfig` from Yaml"""
         cls.validate(tuple(node_key.value for node_key, _ in node.value))
         for node_key, node_value in node.value:
@@ -223,19 +231,77 @@ class PluginConfig(yaml.YAMLObject):
         next_page_url_key_path: Union[str, JSONPath]
         next_page_query_obj_key_path: Union[str, JSONPath]
         next_page_merge_key_path: Union[str, JSONPath]
+        count_tpl: str
         next_page_url_tpl: str
         next_page_query_obj: str
         count_endpoint: str
         start_page: int
 
+    class Sort(TypedDict):
+        """Configuration for sort during search"""
+
+        sort_by_default: List[Tuple[str, str]]
+        sort_by_tpl: str
+        sort_param_mapping: Dict[str, str]
+        sort_order_mapping: Dict[Literal["ascending", "descending"], str]
+        max_sort_params: Annotated[int, Gt(0)]
+
+    class OrderOnResponse(TypedDict):
+        """Configuration for order on-response during download"""
+
+        metadata_mapping: Dict[str, Union[str, List[str]]]
+
+    class OrderStatusSuccess(TypedDict):
+        """
+        Configuration to identify order status success during download
+
+        Order status response matching the following parameters are considered success
+        At least one is required
+        """
+
+        status: Annotated[str, Doc("Variable in the order status response json body")]
+        message: Annotated[str, Doc("Variable in the order status response json body")]
+        http_code: Annotated[int, Doc("HTTP code of the order status response")]
+
+    class OrderStatusOrdered(TypedDict):
+        """
+        Configuration to identify order status ordered during download
+        """
+
+        http_code: Annotated[int, Doc("HTTP code of the order status response")]
+
+    class OrderStatusRequest(TypedDict):
+        """
+        Order status request configuration
+        """
+
+        method: Annotated[str, Doc("Request HTTP method")]
+        headers: Annotated[Dict[str, Any], Doc("Request hearders")]
+
     class OrderStatusOnSuccess(TypedDict):
-        """Configuration for order on-success during download"""
+        """Configuration for order status on-success during download"""
 
-        need_search: bool
+        need_search: Annotated[bool, Doc("If a new search is needed on success")]
         result_type: str
         results_entry: str
         metadata_mapping: Dict[str, Union[str, List[str]]]
 
+    class OrderStatus(TypedDict):
+        """Configuration for order status during download"""
+
+        request: PluginConfig.OrderStatusRequest
+        metadata_mapping: Annotated[
+            Dict[str, Union[str, List[str]]],
+            Doc("Metadata-mapping used to parse order status response"),
+        ]
+        success: PluginConfig.OrderStatusSuccess
+        error: Annotated[
+            Dict[str, Any],
+            Doc("Part of the order status response that tells there is an error"),
+        ]
+        ordered: PluginConfig.OrderStatusOrdered
+        on_success: PluginConfig.OrderStatusOnSuccess
+
     name: str
     type: str
 
@@ -251,12 +317,15 @@ class PluginConfig(yaml.YAMLObject):
     result_type: str
     results_entry: str
     pagination: PluginConfig.Pagination
+    sort: PluginConfig.Sort
     query_params_key: str
-    discover_metadata: Dict[str, str]
+    discover_metadata: Dict[str, Union[str, bool]]
     discover_product_types: Dict[str, Any]
     discover_queryables: Dict[str, Any]
     metadata_mapping: Dict[str, Union[str, List[str]]]
     free_params: Dict[Any, Any]
+    constraints_file_url: str
+    remove_from_queryables: List[str]
     free_text_search_operations: Dict[str, Any]  # ODataV4Search
     metadata_pre_mapping: Dict[str, Any]  # ODataV4Search
     data_request_url: str  # DataRequestSearch
@@ -268,16 +337,30 @@ class PluginConfig(yaml.YAMLObject):
     max_connections: int  # StaticStacSearch
     timeout: float  # StaticStacSearch
     s3_bucket: str  # CreodiasS3Search
+    end_date_excluded: bool  # BuildSearchResult
+    remove_from_query: List[str]  # BuildSearchResult
+    ssl_verify: bool
 
     # download -------------------------------------------------------------------------
     base_uri: str
     outputs_prefix: str
     extract: bool
+    outputs_extension: str
     order_enabled: bool  # HTTPDownload
     order_method: str  # HTTPDownload
     order_headers: Dict[str, str]  # HTTPDownload
-    order_status_on_success: PluginConfig.OrderStatusOnSuccess
+
+    order_on_response: PluginConfig.OrderOnResponse
+    order_status: PluginConfig.OrderStatus
+    no_auth_download: Annotated[
+        bool,
+        Doc(
+            "Do not authenticate the download request but only the order and order status ones."
+        ),
+    ]
     bucket_path_level: int  # S3RestDownload
+    requester_pays: bool  # AwsDownload
+    flatten_top_dirs: bool
 
     # auth -----------------------------------------------------------------------------
     credentials: Dict[str, str]
@@ -299,7 +382,13 @@ class PluginConfig(yaml.YAMLObject):
     token_exchange_post_data_method: str  # OIDCAuthorizationCodeFlowAuth
     token_uri: str  # OIDCAuthorizationCodeFlowAuth
     token_key: str  # OIDCAuthorizationCodeFlowAuth
+    req_data: Dict[str, Any]  # TokenAuth
     signed_url_key: str  # SASAuth
+    refresh_uri: str  # TokenAuth
+    refresh_token_key: str  # TokenAuth
+    subject: Dict[str, Any]  # TokenExchangeAuth
+    subject_issuer: str  # TokenExchangeAuth
+    audience: str  # TokenExchangeAuth
 
     yaml_loader = yaml.Loader
     yaml_dumper = yaml.SafeDumper
@@ -362,7 +451,7 @@ def load_config(config_path: str) -> Dict[str, ProviderConfig]:
     :returns: The default provider's configuration
     :rtype: dict
     """
-    logger.debug(f"Loading configuration from {config_path}")
+    logger.debug("Loading configuration from %s", config_path)
     config: Dict[str, ProviderConfig] = {}
     try:
         # Providers configs are stored in this file as separated yaml documents

eodag/plugins/apis/base.py

@@ -17,34 +17,16 @@
 # limitations under the License.
 from __future__ import annotations
 
-import logging
-from typing import TYPE_CHECKING, Any, Dict, List, Optional, Tuple
+from eodag.plugins.download.base import Download
+from eodag.plugins.search.base import Search
 
-from pydantic.fields import Field, FieldInfo
 
-if TYPE_CHECKING:
-    from eodag.api.product import EOProduct
-    from eodag.api.search_result import SearchResult
-    from eodag.config import PluginConfig
-    from eodag.utils import DownloadedCallback, ProgressCallback
-
-from eodag.plugins.base import PluginTopic
-from eodag.utils import (
-    DEFAULT_DOWNLOAD_TIMEOUT,
-    DEFAULT_DOWNLOAD_WAIT,
-    DEFAULT_ITEMS_PER_PAGE,
-    DEFAULT_PAGE,
-    Annotated,
-)
-
-logger = logging.getLogger("eodag.apis.base")
-
-
-class Api(PluginTopic):
+class Api(Search, Download):
     """Plugins API Base plugin
 
-    An Api plugin has three download methods that it must implement:
+    An Api plugin inherit the methods from Search and Download plugins.
 
+    There are three methods that it must implement:
     - ``query``: search for products
     - ``download``: download a single :class:`~eodag.api.product._product.EOProduct`
     - ``download_all``: download multiple products from a :class:`~eodag.api.search_result.SearchResult`
@@ -62,148 +44,12 @@ class Api(PluginTopic):
       (e.g. 'file:///tmp/product_folder' on Linux or
       'file:///C:/Users/username/AppData/LOcal/Temp' on Windows)
     - save a *record* file in the directory ``outputs_prefix/.downloaded`` whose name
-      is built on the MD5 hash of the product's ``remote_location`` attribute
-      (``hashlib.md5(remote_location.encode("utf-8")).hexdigest()``) and whose content is
-      the product's ``remote_location`` attribute itself.
+      is built on the MD5 hash of the product's ``product_type`` and ``properties['id']``
+      attributes (``hashlib.md5((product.product_type+"-"+product.properties['id']).encode("utf-8")).hexdigest()``)
+      and whose content is the product's ``remote_location`` attribute itself.
     - not try to download a product whose ``location`` attribute already points to an
       existing file/directory
     - not try to download a product if its *record* file exists as long as the expected
       product's file/directory. If the *record* file only is found, it must be deleted
       (it certainly indicates that the download didn't complete)
     """
-
-    def clear(self) -> None:
-        """Method used to clear a search context between two searches."""
-        pass
-
-    def query(
-        self,
-        product_type: Optional[str] = None,
-        items_per_page: int = DEFAULT_ITEMS_PER_PAGE,
-        page: int = DEFAULT_PAGE,
-        count: bool = True,
-        **kwargs: Any,
-    ) -> Tuple[List[EOProduct], Optional[int]]:
-        """Implementation of how the products must be searched goes here.
-
-        This method must return a tuple with (1) a list of EOProduct instances (see eodag.api.product module)
-        which will be processed by a Download plugin (2) and the total number of products matching
-        the search criteria. If ``count`` is False, the second element returned must be ``None``.
-        """
-        raise NotImplementedError("A Api plugin must implement a method named query")
-
-    def discover_product_types(self) -> Optional[Dict[str, Any]]:
-        """Fetch product types list from provider using `discover_product_types` conf"""
-        return None
-
-    def discover_queryables(
-        self, **kwargs: Any
-    ) -> Optional[Dict[str, Annotated[Any, FieldInfo]]]:
-        """Fetch queryables list from provider using `discover_queryables` conf
-
-        :param kwargs: additional filters for queryables (`productType` and other search
-                       arguments)
-        :type kwargs: Any
-        :returns: fetched queryable parameters dict
-        :rtype: Optional[Dict[str, Annotated[Any, FieldInfo]]]
-        """
-        return None
-
-    def get_defaults_as_queryables(
-        self, product_type: str
-    ) -> Dict[str, Annotated[Any, FieldInfo]]:
-        """
-        Return given product type defaut settings as queryables
-
-        :param product_type: given product type
-        :type product_type: str
-        :returns: queryable parameters dict
-        :rtype: Dict[str, Annotated[Any, FieldInfo]]
-        """
-        defaults = self.config.products.get(product_type, {})
-        queryables = {}
-        for parameter, value in defaults.items():
-            queryables[parameter] = Annotated[type(value), Field(default=value)]
-        return queryables
-
-    def download(
-        self,
-        product: EOProduct,
-        auth: Optional[PluginConfig] = None,
-        progress_callback: Optional[ProgressCallback] = None,
-        wait: int = DEFAULT_DOWNLOAD_WAIT,
-        timeout: int = DEFAULT_DOWNLOAD_TIMEOUT,
-        **kwargs: Any,
-    ) -> Optional[str]:
-        """
-        Base download method. Not available, it must be defined for each plugin.
-
-        :param product: The EO product to download
-        :type product: :class:`~eodag.api.product._product.EOProduct`
-        :param auth: (optional) The configuration of a plugin of type Authentication
-        :type auth: :class:`~eodag.config.PluginConfig`
-        :param progress_callback: (optional) A progress callback
-        :type progress_callback: :class:`~eodag.utils.ProgressCallback`
-        :param wait: (optional) If download fails, wait time in minutes between two download tries
-        :type wait: int
-        :param timeout: (optional) If download fails, maximum time in minutes before stop retrying
-                        to download
-        :type timeout: int
-        :param kwargs: `outputs_prefix` (str), `extract` (bool), `delete_archive` (bool)
-                        and `dl_url_params` (dict) can be provided as additional kwargs
-                        and will override any other values defined in a configuration
-                        file or with environment variables.
-        :type kwargs: Union[str, bool, dict]
-        :returns: The absolute path to the downloaded product in the local filesystem
-            (e.g. '/tmp/product.zip' on Linux or
-            'C:\\Users\\username\\AppData\\Local\\Temp\\product.zip' on Windows)
-        :rtype: str
-        """
-        raise NotImplementedError(
-            "An Api plugin must implement a method named download"
-        )
-
-    def download_all(
-        self,
-        products: SearchResult,
-        auth: Optional[PluginConfig] = None,
-        downloaded_callback: Optional[DownloadedCallback] = None,
-        progress_callback: Optional[ProgressCallback] = None,
-        wait: int = DEFAULT_DOWNLOAD_WAIT,
-        timeout: int = DEFAULT_DOWNLOAD_TIMEOUT,
-        **kwargs: Any,
-    ) -> List[str]:
-        """
-        Base download_all method.
-
-        :param products: Products to download
-        :type products: :class:`~eodag.api.search_result.SearchResult`
-        :param auth: (optional) The configuration of a plugin of type Authentication
-        :type auth: :class:`~eodag.config.PluginConfig`
-        :param downloaded_callback: (optional) A method or a callable object which takes
-                                    as parameter the ``product``. You can use the base class
-                                    :class:`~eodag.api.product.DownloadedCallback` and override
-                                    its ``__call__`` method. Will be called each time a product
-                                    finishes downloading
-        :type downloaded_callback: Callable[[:class:`~eodag.api.product._product.EOProduct`], None]
-                                   or None
-        :param progress_callback: (optional) A progress callback
-        :type progress_callback: :class:`~eodag.utils.ProgressCallback`
-        :param wait: (optional) If download fails, wait time in minutes between two download tries
-        :type wait: int
-        :param timeout: (optional) If download fails, maximum time in minutes before stop retrying
-                        to download
-        :type timeout: int
-        :param kwargs: `outputs_prefix` (str), `extract` (bool), `delete_archive` (bool)
-                        and `dl_url_params` (dict) can be provided as additional kwargs
-                        and will override any other values defined in a configuration
-                        file or with environment variables.
-        :type kwargs: Union[str, bool, dict]
-        :returns: List of absolute paths to the downloaded products in the local
-            filesystem (e.g. ``['/tmp/product.zip']`` on Linux or
-            ``['C:\\Users\\username\\AppData\\Local\\Temp\\product.zip']`` on Windows)
-        :rtype: list
-        """
-        raise NotImplementedError(
-            "A Api plugin must implement a method named download_all"
-        )

eodag/plugins/apis/ecmwf.py

@@ -18,42 +18,47 @@
 from __future__ import annotations
 
 import logging
-from datetime import datetime
-from typing import TYPE_CHECKING, Any, Dict, List, Optional, Tuple
+import os
+from datetime import datetime, timezone
+from typing import TYPE_CHECKING
 
 import geojson
 from ecmwfapi import ECMWFDataServer, ECMWFService
 from ecmwfapi.api import APIException, Connection, get_apikey_values
 
 from eodag.plugins.apis.base import Api
-from eodag.plugins.download.base import Download
+from eodag.plugins.search import PreparedSearch
 from eodag.plugins.search.base import Search
 from eodag.plugins.search.build_search_result import BuildPostSearchResult
-from eodag.rest.stac import DEFAULT_MISSION_START_DATE
 from eodag.utils import (
     DEFAULT_DOWNLOAD_TIMEOUT,
     DEFAULT_DOWNLOAD_WAIT,
-    DEFAULT_ITEMS_PER_PAGE,
-    DEFAULT_PAGE,
+    DEFAULT_MISSION_START_DATE,
     get_geometry_from_various,
     path_to_uri,
+    sanitize,
     urlsplit,
 )
 from eodag.utils.exceptions import AuthenticationError, DownloadError
 from eodag.utils.logging import get_logging_verbose
 
 if TYPE_CHECKING:
+    from typing import Any, Dict, List, Optional, Tuple, Union
+
+    from requests.auth import AuthBase
+
     from eodag.api.product import EOProduct
     from eodag.api.search_result import SearchResult
     from eodag.config import PluginConfig
-    from eodag.utils import DownloadedCallback, ProgressCallback
+    from eodag.types.download_args import DownloadConf
+    from eodag.utils import DownloadedCallback, ProgressCallback, Unpack
 
 logger = logging.getLogger("eodag.apis.ecmwf")
 
 ECMWF_MARS_KNOWN_FORMATS = {"grib": "grib", "netcdf": "nc"}
 
 
-class EcmwfApi(Download, Api, BuildPostSearchResult):
+class EcmwfApi(Api, BuildPostSearchResult):
     """A plugin that enables to build download-request and download data on ECMWF MARS.
 
     Builds a single ready-to-download :class:`~eodag.api.product._product.EOProduct`
@@ -84,10 +89,7 @@ class EcmwfApi(Download, Api, BuildPostSearchResult):
 
     def query(
         self,
-        product_type: Optional[str] = None,
-        items_per_page: int = DEFAULT_ITEMS_PER_PAGE,
-        page: int = DEFAULT_PAGE,
-        count: bool = True,
+        prep: PreparedSearch = PreparedSearch(),
         **kwargs: Any,
     ) -> Tuple[List[EOProduct], Optional[int]]:
         """Build ready-to-download SearchResult"""
@@ -112,7 +114,7 @@ class EcmwfApi(Download, Api, BuildPostSearchResult):
         if "completionTimeFromAscendingNode" not in kwargs:
             kwargs["completionTimeFromAscendingNode"] = getattr(
                 self.config, "product_type_config", {}
-            ).get("missionEndDate", None) or datetime.utcnow().isoformat(
+            ).get("missionEndDate", None) or datetime.now(timezone.utc).isoformat(
                 timespec="seconds"
             )
 
@@ -120,9 +122,7 @@ class EcmwfApi(Download, Api, BuildPostSearchResult):
         if "geometry" in kwargs:
             kwargs["geometry"] = get_geometry_from_various(geometry=kwargs["geometry"])
 
-        return BuildPostSearchResult.query(
-            self, items_per_page=items_per_page, page=page, count=count, **kwargs
-        )
+        return BuildPostSearchResult.query(self, prep, **kwargs)
 
     def authenticate(self) -> Dict[str, Optional[str]]:
         """Check credentials and returns information needed for auth
@@ -156,21 +156,23 @@ class EcmwfApi(Download, Api, BuildPostSearchResult):
     def download(
         self,
         product: EOProduct,
-        auth: Optional[PluginConfig] = None,
+        auth: Optional[Union[AuthBase, Dict[str, str]]] = None,
         progress_callback: Optional[ProgressCallback] = None,
         wait: int = DEFAULT_DOWNLOAD_WAIT,
         timeout: int = DEFAULT_DOWNLOAD_TIMEOUT,
-        **kwargs: Any,
+        **kwargs: Unpack[DownloadConf],
     ) -> Optional[str]:
         """Download data from ECMWF MARS"""
         product_format = product.properties.get("format", "grib")
         product_extension = ECMWF_MARS_KNOWN_FORMATS.get(product_format, product_format)
+        kwargs["outputs_extension"] = kwargs.get(
+            "outputs_extension", f".{product_extension}"
+        )
 
         # Prepare download
         fs_path, record_filename = self._prepare_download(
             product,
             progress_callback=progress_callback,
-            outputs_extension=f".{product_extension}",
             **kwargs,
         )
 
@@ -179,6 +181,13 @@ class EcmwfApi(Download, Api, BuildPostSearchResult):
                 product.location = path_to_uri(fs_path)
             return fs_path
 
+        new_fs_path = os.path.join(
+            os.path.dirname(fs_path), sanitize(product.properties["title"])
+        )
+        if not os.path.isdir(new_fs_path):
+            os.makedirs(new_fs_path)
+        fs_path = os.path.join(new_fs_path, os.path.basename(fs_path))
+
         # get download request dict from product.location/downloadLink url query string
         # separate url & parameters
         download_request = geojson.loads(urlsplit(product.location).query)
@@ -222,13 +231,12 @@ class EcmwfApi(Download, Api, BuildPostSearchResult):
             fh.write(product.properties["downloadLink"])
         logger.debug("Download recorded in %s", record_filename)
 
-        # do not try to extract or delete grib/netcdf
+        # do not try to extract a directory
         kwargs["extract"] = False
 
         product_path = self._finalize(
-            fs_path,
+            new_fs_path,
             progress_callback=progress_callback,
-            outputs_extension=f".{product_extension}",
             **kwargs,
         )
         product.location = path_to_uri(product_path)
@@ -237,12 +245,12 @@ class EcmwfApi(Download, Api, BuildPostSearchResult):
     def download_all(
         self,
         products: SearchResult,
-        auth: Optional[PluginConfig] = None,
+        auth: Optional[Union[AuthBase, Dict[str, str]]] = None,
         downloaded_callback: Optional[DownloadedCallback] = None,
         progress_callback: Optional[ProgressCallback] = None,
         wait: int = DEFAULT_DOWNLOAD_WAIT,
         timeout: int = DEFAULT_DOWNLOAD_TIMEOUT,
-        **kwargs: Any,
+        **kwargs: Unpack[DownloadConf],
     ) -> List[str]:
         """
         Download all using parent (base plugin) method
@@ -256,3 +264,7 @@ class EcmwfApi(Download, Api, BuildPostSearchResult):
             timeout=timeout,
             **kwargs,
         )
+
+    def clear(self) -> None:
+        """Clear search context"""
+        pass