eodag 3.10.1__py3-none-any.whl → 4.0.0a2__py3-none-any.whl

eodag/config.py

@@ -38,7 +38,6 @@ from typing import (
 import orjson
 import requests
 import yaml
-import yaml.constructor
 import yaml.parser
 from annotated_types import Gt
 from jsonpath_ng import JSONPath
@@ -64,8 +63,8 @@ from eodag.utils.exceptions import ValidationError
 
 logger = logging.getLogger("eodag.config")
 
-EXT_PRODUCT_TYPES_CONF_URI = (
-    "https://cs-si.github.io/eodag/eodag/resources/ext_product_types.json"
+EXT_COLLECTIONS_CONF_URI = (
+    "https://cs-si.github.io/eodag/eodag/resources/ext_collections.json"
 )
 AUTH_TOPIC_KEYS = ("auth", "search_auth", "download_auth")
 PLUGINS_TOPICS_KEYS = ("api", "search", "download") + AUTH_TOPIC_KEYS
@@ -114,7 +113,7 @@ class ProviderConfig(yaml.YAMLObject):
                      Lower value means lower priority. (Default: 0)
     :param api: (optional) The configuration of a plugin of type Api
     :param search: (optional) The configuration of a plugin of type Search
-    :param products: (optional) The products types supported by the provider
+    :param products: (optional) The collections supported by the provider
     :param download: (optional) The configuration of a plugin of type Download
     :param auth: (optional) The configuration of a plugin of type Authentication
     :param search_auth: (optional) The configuration of a plugin of type Authentication for search
@@ -135,7 +134,7 @@ class ProviderConfig(yaml.YAMLObject):
     auth: PluginConfig
     search_auth: PluginConfig
     download_auth: PluginConfig
-    product_types_fetched: bool  # set in core.update_product_types_list
+    collections_fetched: bool  # set in core.update_collections_list
 
     yaml_loader = yaml.Loader
     yaml_dumper = yaml.SafeDumper
@@ -241,10 +240,15 @@ class PluginConfig(yaml.YAMLObject):
         next_page_url_tpl: str
         #: The query-object for POST pagination requests.
         next_page_query_obj: str
+        #: Next page token key used in pagination. Can be guessed from ``KNOWN_NEXT_PAGE_TOKEN_KEYS`` (but needed by
+        # ``stac-fastapi-eodag`` that cannot guess and will use ``page`` as default).
+        next_page_token_key: str
         #: The endpoint for counting the number of items satisfying a request
         count_endpoint: str
         #: Index of the starting page
         start_page: int
+        #: Key in the current page URL for the next page URL
+        parse_url_key: str
 
     class Sort(TypedDict):
         """Configuration for sort during search"""
@@ -274,17 +278,19 @@ class PluginConfig(yaml.YAMLObject):
         metadata_path: str
         #: list search parameters to send as is to the provider
         search_param_unparsed: list[str]
+        #: Use as STAC extension prefix if it does not have one already
+        metadata_prefix: str
         #: Whether an error must be raised when using a search parameter which is not queryable or not
         raise_mtd_discovery_error: bool
 
-    class DiscoverProductTypes(TypedDict, total=False):
-        """Configuration for product types discovery"""
+    class DiscoverCollections(TypedDict, total=False):
+        """Configuration for collections discovery"""
 
-        #: URL from which the product types can be fetched
+        #: URL from which the collections can be fetched
         fetch_url: Optional[str]
-        #: HTTP method used to fetch product types
+        #: HTTP method used to fetch collections
         fetch_method: str
-        #: Request body to fetch product types using POST method
+        #: Request body to fetch collections using POST method
         fetch_body: dict[str, Any]
         #: Maximum number of connections for concurrent HTTP requests
         max_connections: int
@@ -294,32 +300,32 @@ class PluginConfig(yaml.YAMLObject):
         start_page: int
         #: Type of the provider result
         result_type: str
-        #: JsonPath to the list of product types
+        #: JsonPath to the list of collections
         results_entry: Union[JSONPath, str]
-        #: Mapping for the product type id
-        generic_product_type_id: str
-        #: Mapping for product type metadata (e.g. ``abstract``, ``licence``) which can be parsed from the provider
+        #: Mapping for the collection id
+        generic_collection_id: str
+        #: Mapping for collection metadata (e.g. ``description``, ``license``) which can be parsed from the provider
         #: result
-        generic_product_type_parsable_metadata: dict[str, str]
-        #: Mapping for product type properties which can be parsed from the result and are not product type metadata
-        generic_product_type_parsable_properties: dict[str, str]
-        #: Mapping for product type properties which cannot be parsed from the result and are not product type metadata
-        generic_product_type_unparsable_properties: dict[str, str]
+        generic_collection_parsable_metadata: dict[str, str]
+        #: Mapping for collection properties which can be parsed from the result and are not collection metadata
+        generic_collection_parsable_properties: dict[str, str]
+        #: Mapping for collection properties which cannot be parsed from the result and are not collection metadata
+        generic_collection_unparsable_properties: dict[str, str]
         #: URL to fetch data for a single collection
         single_collection_fetch_url: str
         #: Query string to be added to the fetch_url to filter for a collection
         single_collection_fetch_qs: str
-        #: Mapping for product type metadata returned by the endpoint given in single_collection_fetch_url. If ``ID``
-        #: is redefined in this mapping, it will replace ``generic_product_type_id`` value
-        single_product_type_parsable_metadata: dict[str, str]
+        #: Mapping for collection metadata returned by the endpoint given in single_collection_fetch_url. If ``id``
+        #: is redefined in this mapping, it will replace ``generic_collection_id`` value
+        single_collection_parsable_metadata: dict[str, str]
 
     class DiscoverQueryables(TypedDict, total=False):
         """Configuration for queryables discovery"""
 
-        #: URL to fetch the queryables valid for all product types
+        #: URL to fetch the queryables valid for all collections
         fetch_url: Optional[str]
-        #: URL to fetch the queryables for a specific product type
-        product_type_fetch_url: Optional[str]
+        #: URL to fetch the queryables for a specific collection
+        collection_fetch_url: Optional[str]
         #: Type of the result
         result_type: str
         #: JsonPath to retrieve the queryables from the provider result
@@ -329,6 +335,30 @@ class PluginConfig(yaml.YAMLObject):
         #: :class:`~eodag.plugins.search.base.Search` Key in the json result where the constraints can be found
         constraints_entry: str
 
+    class CollectionSelector(TypedDict, total=False):
+        """Define the criteria to select a collection in :class:`~eodag.config.PluginConfig.DynamicDiscoverQueryables`.
+
+        The selector matches if the field value starts with the given prefix,
+        i.e. it matches if ``parameters[field].startswith(prefix)==True``"""
+
+        #: Field in the search parameters to match
+        field: str
+        #: Prefix to match in the field
+        prefix: str
+
+    class DynamicDiscoverQueryables(TypedDict, total=False):
+        """Configuration for queryables dynamic discovery.
+
+        The given configuration for queryables discovery is used if any collection selector
+        matches the search parameters.
+        """
+
+        #: List of collection selection criterias
+        collection_selector: list[PluginConfig.CollectionSelector]
+
+        #: Configuration for queryables discovery to use
+        discover_queryables: PluginConfig.DiscoverQueryables
+
     class OrderOnResponse(TypedDict):
         """Configuration for order on-response during download"""
 
@@ -431,8 +461,8 @@ class PluginConfig(yaml.YAMLObject):
     # search & api -----------------------------------------------------------------------------------------------------
     # copied from ProviderConfig in PluginManager.get_search_plugins()
     priority: int
-    # per product type metadata-mapping, set in core._prepare_search
-    product_type_config: dict[str, Any]
+    # per collection metadata-mapping, set in core._prepare_search
+    collection_config: dict[str, Any]
 
     #: :class:`~eodag.plugins.search.base.Search` Plugin API endpoint
     api_endpoint: str
@@ -449,14 +479,14 @@ class PluginConfig(yaml.YAMLObject):
     sort: PluginConfig.Sort
     #: :class:`~eodag.plugins.search.base.Search` Configuration for the metadata auto-discovery
     discover_metadata: PluginConfig.DiscoverMetadata
-    #: :class:`~eodag.plugins.search.base.Search` Configuration for the product types auto-discovery
-    discover_product_types: PluginConfig.DiscoverProductTypes
+    #: :class:`~eodag.plugins.search.base.Search` Configuration for the collections auto-discovery
+    discover_collections: PluginConfig.DiscoverCollections
     #: :class:`~eodag.plugins.search.base.Search` Configuration for the queryables auto-discovery
     discover_queryables: PluginConfig.DiscoverQueryables
     #: :class:`~eodag.plugins.search.base.Search` The mapping between eodag metadata and the plugin specific metadata
     metadata_mapping: dict[str, Union[str, list[str]]]
     #: :class:`~eodag.plugins.search.base.Search` :attr:`~eodag.config.PluginConfig.metadata_mapping` got from the given
-    #: product type
+    #: collection
     metadata_mapping_from_product: str
     #: :class:`~eodag.plugins.search.base.Search` A mapping for the metadata of individual assets
     assets_mapping: dict[str, dict[str, Any]]
@@ -476,26 +506,19 @@ class PluginConfig(yaml.YAMLObject):
     per_product_metadata_query: bool
     #: :class:`~eodag.plugins.search.qssearch.ODataV4Search` Dict used to simplify further metadata extraction
     metadata_pre_mapping: PluginConfig.MetadataPreMapping
-    #: :class:`~eodag.plugins.search.data_request_search.DataRequestSearch` URL to which the data request shall be sent
-    data_request_url: str
-    #: :class:`~eodag.plugins.search.data_request_search.DataRequestSearch` URL to fetch the status of the data request
-    status_url: str
-    #: :class:`~eodag.plugins.search.data_request_search.DataRequestSearch`
-    #: URL to fetch the search result when the data request is done
-    result_url: str
-    #: :class:`~eodag.plugins.search.data_request_search.DataRequestSearch`
-    #: if date parameters are mandatory in the request
-    dates_required: bool
     #: :class:`~eodag.plugins.search.csw.CSWSearch` Search definition dictionary
     search_definition: dict[str, Any]
     #: :class:`~eodag.plugins.search.qssearch.PostJsonSearch` Whether to merge responses or not (`aws_eos` specific)
     merge_responses: bool
     #: :class:`~eodag.plugins.search.qssearch.PostJsonSearch` Collections names (`aws_eos` specific)
-    collection: list[str]
+    _collection: list[str]
     #: :class:`~eodag.plugins.search.static_stac_search.StaticStacSearch`
     #: Maximum number of connections for concurrent HTTP requests
     max_connections: int
     #: :class:`~eodag.plugins.search.build_search_result.ECMWFSearch`
+    #: if date parameters are mandatory in the request
+    dates_required: bool
+    #: :class:`~eodag.plugins.search.build_search_result.ECMWFSearch`
     #: Whether end date should be excluded from search request or not
     end_date_excluded: bool
     #: :class:`~eodag.plugins.search.build_search_result.ECMWFSearch`
@@ -506,6 +529,12 @@ class PluginConfig(yaml.YAMLObject):
     version: str
     #: :class:`~eodag.plugins.apis.ecmwf.EcmwfApi` url of the authentication endpoint
     auth_endpoint: str
+    #: :class:`~eodag.plugins.search.build_search_result.WekeoECMWFSearch`
+    #: Configurations for the queryables dynamic auto-discovery.
+    #: A configuration is used based on the given selection criterias. The first match is used.
+    #: If no match is found, it falls back to standard behaviors (e.g. discovery using
+    #: :attr:`~eodag.config.PluginConfig.discover_queryables`).
+    dynamic_discover_queryables: list[PluginConfig.DynamicDiscoverQueryables]
 
     # download ---------------------------------------------------------------------------------------------------------
     #: :class:`~eodag.plugins.download.base.Download` Default endpoint url
@@ -521,9 +550,10 @@ class PluginConfig(yaml.YAMLObject):
     flatten_top_dirs: bool
     #: :class:`~eodag.plugins.download.base.Download` Level in extracted path tree where to find data
     archive_depth: int
-    #: :class:`~eodag.plugins.download.base.Download` Whether ignore assets and download using ``downloadLink`` or not
+    #: :class:`~eodag.plugins.download.base.Download` Whether ignore assets and download using ``eodag:download_link``
+    #: or not
     ignore_assets: bool
-    #: :class:`~eodag.plugins.download.base.Download` Product type specific configuration
+    #: :class:`~eodag.plugins.download.base.Download` Collection specific configuration
     products: dict[str, dict[str, Any]]
     #: :class:`~eodag.plugins.download.http.HTTPDownload` Whether the product has to be ordered to download it or not
     order_enabled: bool
@@ -542,7 +572,7 @@ class PluginConfig(yaml.YAMLObject):
     no_auth_download: bool
     #: :class:`~eodag.plugins.download.http.HTTPDownload` Parameters to be added to the query params of the request
     dl_url_params: dict[str, str]
-    #: :class:`~eodag.plugins.download.s3rest.S3RestDownload`
+    #: :class:`~eodag.plugins.download.aws.AwsDownload`
     #: At which level of the path part of the url the bucket can be found
     bucket_path_level: int
     #: :class:`~eodag.plugins.download.aws.AwsDownload` S3 endpoint
@@ -1069,15 +1099,15 @@ def load_stac_provider_config() -> dict[str, Any]:
     ).source
 
 
-def get_ext_product_types_conf(
-    conf_uri: str = EXT_PRODUCT_TYPES_CONF_URI,
+def get_ext_collections_conf(
+    conf_uri: str = EXT_COLLECTIONS_CONF_URI,
 ) -> dict[str, Any]:
-    """Read external product types conf
+    """Read external collections conf
 
     :param conf_uri: URI to local or remote configuration file
-    :returns: The external product types configuration
+    :returns: The external collections configuration
     """
-    logger.info("Fetching external product types from %s", conf_uri)
+    logger.info("Fetching external collections from %s", conf_uri)
     if conf_uri.lower().startswith("http"):
         # read from remote
         try:
@@ -1089,7 +1119,7 @@ def get_ext_product_types_conf(
         except requests.RequestException as e:
             logger.debug(e)
             logger.warning(
-                "Could not read remote external product types conf from %s", conf_uri
+                "Could not read remote external collections conf from %s", conf_uri
             )
             return {}
     elif conf_uri.lower().startswith("file"):
@@ -1102,6 +1132,6 @@ def get_ext_product_types_conf(
     except (orjson.JSONDecodeError, FileNotFoundError) as e:
         logger.debug(e)
         logger.warning(
-            "Could not read local external product types conf from %s", conf_uri
+            "Could not read local external collections conf from %s", conf_uri
         )
         return {}

eodag/plugins/apis/base.py

@@ -45,8 +45,8 @@ class Api(Search, Download):
       (e.g. 'file:///tmp/product_folder' on Linux or
       'file:///C:/Users/username/AppData/Local/Temp' on Windows)
     - save a *record* file in the directory ``output_dir/.downloaded`` whose name
-      is built on the MD5 hash of the product's ``product_type`` and ``properties['id']``
-      attributes (``hashlib.md5((product.product_type+"-"+product.properties['id']).encode("utf-8")).hexdigest()``)
+      is built on the MD5 hash of the product's ``collection`` and ``properties['id']``
+      attributes (``hashlib.md5((product.collection+"-"+product.properties['id']).encode("utf-8")).hexdigest()``)
       and whose content is the product's ``remote_location`` attribute itself.
     - not try to download a product whose ``location`` attribute already points to an
       existing file/directory

eodag/plugins/apis/ecmwf.py

@@ -27,6 +27,7 @@ from ecmwfapi import ECMWFDataServer, ECMWFService
 from ecmwfapi.api import APIException, Connection, get_apikey_values
 from pydantic.fields import FieldInfo
 
+from eodag.api.search_result import RawSearchResult
 from eodag.plugins.apis.base import Api
 from eodag.plugins.search import PreparedSearch
 from eodag.plugins.search.base import Search
@@ -35,6 +36,7 @@ from eodag.utils import (
     DEFAULT_DOWNLOAD_TIMEOUT,
     DEFAULT_DOWNLOAD_WAIT,
     DEFAULT_MISSION_START_DATE,
+    get_collection_dates,
     get_geometry_from_various,
     path_to_uri,
     sanitize,
@@ -52,7 +54,6 @@ if TYPE_CHECKING:
     from eodag.api.product import EOProduct
     from eodag.api.search_result import SearchResult
     from eodag.config import PluginConfig
-    from eodag.types import S3SessionKwargs
     from eodag.types.download_args import DownloadConf
     from eodag.utils import DownloadedCallback, ProgressCallback, Unpack
 
@@ -103,36 +104,45 @@ class EcmwfApi(Api, ECMWFSearch):
         self.config.__dict__.setdefault("pagination", {"next_page_query_obj": "{{}}"})
         self.config.__dict__.setdefault("api_endpoint", "")
 
-    def do_search(self, *args: Any, **kwargs: Any) -> list[dict[str, Any]]:
+    def do_search(
+        self, prep: PreparedSearch = PreparedSearch(items_per_page=None), **kwargs: Any
+    ) -> RawSearchResult:
         """Should perform the actual search request."""
-        return [{}]
+        raw_search_results = RawSearchResult([{}])
+        raw_search_results.search_params = kwargs
+        raw_search_results.query_params = (
+            prep.query_params if hasattr(prep, "query_params") else {}
+        )
+        raw_search_results.collection_def_params = (
+            prep.collection_def_params if hasattr(prep, "collection_def_params") else {}
+        )
+        return raw_search_results
 
     def query(
         self,
         prep: PreparedSearch = PreparedSearch(),
         **kwargs: Any,
-    ) -> tuple[list[EOProduct], Optional[int]]:
+    ) -> SearchResult:
         """Build ready-to-download SearchResult"""
 
-        # check productType, dates, geometry, use defaults if not specified
-        # productType
-        if not kwargs.get("productType"):
-            kwargs["productType"] = "%s_%s_%s" % (
+        # check collection, dates, geometry, use defaults if not specified
+        # collection
+        if not kwargs.get("collection"):
+            kwargs["collection"] = "%s_%s_%s" % (
                 kwargs.get("ecmwf:dataset", "mars"),
                 kwargs.get("ecmwf:type", ""),
                 kwargs.get("ecmwf:levtype", ""),
             )
+
+        col_start, col_end = get_collection_dates(
+            getattr(self.config, "collection_config", {})
+        )
         # start date
-        if "startTimeFromAscendingNode" not in kwargs:
-            kwargs["startTimeFromAscendingNode"] = (
-                getattr(self.config, "product_type_config", {}).get("missionStartDate")
-                or DEFAULT_MISSION_START_DATE
-            )
+        if "start_datetime" not in kwargs:
+            kwargs["start_datetime"] = col_start or DEFAULT_MISSION_START_DATE
         # end date
-        if "completionTimeFromAscendingNode" not in kwargs:
-            kwargs["completionTimeFromAscendingNode"] = getattr(
-                self.config, "product_type_config", {}
-            ).get("missionEndDate") or datetime.now(timezone.utc).isoformat(
+        if "end_datetime" not in kwargs:
+            kwargs["end_datetime"] = col_end or datetime.now(timezone.utc).isoformat(
                 timespec="seconds"
             )
 
@@ -173,7 +183,7 @@ class EcmwfApi(Api, ECMWFSearch):
     def download(
         self,
         product: EOProduct,
-        auth: Optional[Union[AuthBase, S3SessionKwargs, S3ServiceResource]] = None,
+        auth: Optional[Union[AuthBase, S3ServiceResource]] = None,
         progress_callback: Optional[ProgressCallback] = None,
         wait: float = DEFAULT_DOWNLOAD_WAIT,
         timeout: float = DEFAULT_DOWNLOAD_TIMEOUT,
@@ -205,7 +215,7 @@ class EcmwfApi(Api, ECMWFSearch):
             os.makedirs(new_fs_path)
         fs_path = os.path.join(new_fs_path, os.path.basename(fs_path))
 
-        # get download request dict from product.location/downloadLink url query string
+        # get download request dict from product.location/eodag:download_link url query string
         # separate url & parameters
         download_request = geojson.loads(urlsplit(product.location).query)
 
@@ -245,7 +255,7 @@ class EcmwfApi(Api, ECMWFSearch):
             raise DownloadError(e)
 
         with open(record_filename, "w") as fh:
-            fh.write(product.properties["downloadLink"])
+            fh.write(product.properties["eodag:download_link"])
         logger.debug("Download recorded in %s", record_filename)
 
         # do not try to extract a directory
@@ -262,7 +272,7 @@ class EcmwfApi(Api, ECMWFSearch):
     def download_all(
         self,
         products: SearchResult,
-        auth: Optional[Union[AuthBase, S3SessionKwargs]] = None,
+        auth: Optional[Union[AuthBase, S3ServiceResource]] = None,
         downloaded_callback: Optional[DownloadedCallback] = None,
         progress_callback: Optional[ProgressCallback] = None,
         wait: float = DEFAULT_DOWNLOAD_WAIT,
@@ -287,13 +297,14 @@ class EcmwfApi(Api, ECMWFSearch):
         pass
 
     def discover_queryables(
-        self, **kwargs: Any
+        self,
+        **kwargs: Any,
     ) -> Optional[dict[str, Annotated[Any, FieldInfo]]]:
         """Fetch queryables list from provider using metadata mapping
 
-        :param kwargs: additional filters for queryables (`productType` and other search
+        :param kwargs: additional filters for queryables (`collection` and other search
                        arguments)
         :returns: fetched queryable parameters dict
         """
-        product_type = kwargs.get("productType")
-        return self.queryables_from_metadata_mapping(product_type)
+        collection = kwargs.get("collection")
+        return self.queryables_from_metadata_mapping(collection)