eodag 3.1.0b1__py3-none-any.whl → 3.1.0b2__py3-none-any.whl

eodag/resources/providers.yml

@@ -1886,7 +1886,7 @@
       "ecmwf:expver": prod
       "ecmwf:type": cf
       "ecmwf:levtype": sfc
-      "ecmwf:origin": ecmf
+      "ecmwf:origin": ecmwf
       "ecmwf:grid": 0.5/0.5
       "ecmwf:param": 59/134/136/146/147/151/165/166/167/168/172/176/177/179/189/235/228002/228039/228139/228141/228144/228164/228228
       "ecmwf:step": 0
@@ -4550,7 +4550,7 @@
         - '{$.Footprint#from_ewkt}'
       # The url to download the product "as is" (literal or as a template to be completed either after the search result
       # is obtained from the provider or during the eodag download phase)
-      downloadLink: 'https://zipper.creodias.eu/odata/v1/Products({uid})/$value'
+      downloadLink: '$.S3Path.`sub(/^(.*)$/, s3:/\\1)`'
       # storageStatus: must be one of ONLINE, STAGING, OFFLINE
       storageStatus: '{$.Online#get_group_name((?P<ONLINE>True)|(?P<OFFLINE>False))}'
       collection:
@@ -4558,8 +4558,6 @@
         - $.null
       quicklook: '$.Assets[?(@.Type="QUICKLOOK")].DownloadLink'
       thumbnail: '$.Assets[?(@.Type="QUICKLOOK")].DownloadLink'
-        # Additional metadata provided by the providers but that don't appear in the reference spec
-      productIdentifier: '$.S3Path'
   download: !plugin
     type: CreodiasS3Download
     s3_endpoint: 'https://eodata.cloudferro.com'
@@ -4785,6 +4783,7 @@
   search: !plugin
     type: ECMWFSearch
     ssl_verify: true
+    dates_required: True
     discover_queryables:
       fetch_url: null
       product_type_fetch_url: null
@@ -5887,6 +5886,9 @@
         completionTimeFromAscendingNode: temporal:endDate
         platformSerialIdentifier: spaceborne:satellitePlatform
         cloudCover: spaceborne:cloudCover
+      sort_order_mapping:
+        ascending: asc
+        descending: desc
     metadata_mapping:
       uid: '$.id'
       # OpenSearch Parameters for Collection Search (Table 3)
@@ -5907,8 +5909,11 @@
       keyword: '$.properties."spaceborne:keywords"'
       # OpenSearch Parameters for Product Search (Table 5)
       orbitNumber:
-        - '{{"query":{{"spaceborne:absoluteOrbitID":{{"eq":"{orbitNumber}"}}}}}}'
+        - '{{"query":{{"spaceborne:absoluteOrbitID":{{"eq":{orbitNumber}}}}}}}'
         - '$.properties."spaceborne:absoluteOrbitID"'
+      relativeOrbitNumber:
+        - '{{"query":{{"spaceborne:orbitID":{{"eq":{relativeOrbitNumber}}}}}}}'
+        - '$.properties."spaceborne:orbitID"'
       orbitDirection:
         - '{{"query":{{"spaceborne:orbitDirection":{{"eq":"{orbitDirection}"}}}}}}'
         - '$.properties."spaceborne:orbitDirection"'
@@ -5916,7 +5921,7 @@
         - '{{"query":{{"spaceborne:swath":{{"eq":"{swathIdentifier}"}}}}}}'
         - '$.properties."spaceborne:swath"'
       cloudCover:
-        - '{{"query":{{"spaceborne:cloudCover":{{"lte":"{cloudCover}"}}}}}}'
+        - '{{"query":{{"spaceborne:cloudCover":{{"lte":{cloudCover}}}}}}}'
         - '$.properties."spaceborne:cloudCover"'
       sensorMode:
         - '{{"query":{{"spaceborne:sensorMode":{{"eq":"{sensorMode}"}}}}}}'
@@ -5982,6 +5987,141 @@
     headers:
       X-API-Key: "{apikey}"
 
+---
+!provider  # MARK: geodes_s3
+  name: geodes_s3
+  priority: 0
+  roles:
+    - host
+  description: French National Space Agency (CNES) Earth Observation portal with internal s3 Datalake
+  url: https://geodes.cnes.fr
+  search: !plugin
+    type: StacListAssets
+    api_endpoint: https://geodes-portal.cnes.fr/api/stac/search
+    s3_endpoint: https://s3.datalake.cnes.fr
+    need_auth: true
+    ssl_verify: false
+    discover_queryables:
+      fetch_url: null
+      product_type_fetch_url: null
+    discover_product_types:
+      fetch_url: https://geodes-portal.cnes.fr/api/stac/collections
+      fetch_method: POST
+      fetch_body:
+        limit: 10000
+    pagination:
+      total_items_nb_key_path: '$.context.matched'
+      # 1000 is ok and 2000 fails
+      max_items_per_page: 1000
+    sort:
+      sort_by_tpl: '{{"sortBy": [ {{"field": "{sort_param}", "direction": "{sort_order}" }} ] }}'
+      sort_by_default:
+        - !!python/tuple [startTimeFromAscendingNode, ASC]
+      sort_param_mapping:
+        id: spaceborne:s2TakeId
+        startTimeFromAscendingNode: temporal:startDate
+        completionTimeFromAscendingNode: temporal:endDate
+        platformSerialIdentifier: spaceborne:satellitePlatform
+        cloudCover: spaceborne:cloudCover
+      sort_order_mapping:
+        ascending: asc
+        descending: desc
+    metadata_mapping:
+      uid: '$.id'
+      # OpenSearch Parameters for Collection Search (Table 3)
+      productType:
+        - '{{"query":{{"dataType":{{"eq":"{productType}"}}}}}}'
+        - '$.properties."dataType"'
+      processingLevel:
+        - '{{"query":{{"spaceborne:productLevel":{{"eq":"{processingLevel}"}}}}}}'
+        - '$.properties."spaceborne:productLevel"'
+      platformSerialIdentifier:
+        - '{{"query":{{"spaceborne:satellitePlatform":{{"eq":"{platformSerialIdentifier}"}}}}}}'
+        - '$.properties."spaceborne:satellitePlatform"'
+      instrument:
+        - '{{"query":{{"spaceborne:satelliteSensor":{{"eq":"{instrument}"}}}}}}'
+        - '$.properties."spaceborne:satelliteSensor"'
+      # INSPIRE obligated OpenSearch Parameters for Collection Search (Table 4)
+      title: '$.properties.identifier'
+      keyword: '$.properties."spaceborne:keywords"'
+      # OpenSearch Parameters for Product Search (Table 5)
+      orbitNumber:
+        - '{{"query":{{"spaceborne:absoluteOrbitID":{{"eq":{orbitNumber}}}}}}}'
+        - '$.properties."spaceborne:absoluteOrbitID"'
+      relativeOrbitNumber:
+        - '{{"query":{{"spaceborne:orbitID":{{"eq":{relativeOrbitNumber}}}}}}}'
+        - '$.properties."spaceborne:orbitID"'
+      orbitDirection:
+        - '{{"query":{{"spaceborne:orbitDirection":{{"eq":"{orbitDirection}"}}}}}}'
+        - '$.properties."spaceborne:orbitDirection"'
+      swathIdentifier:
+        - '{{"query":{{"spaceborne:swath":{{"eq":"{swathIdentifier}"}}}}}}'
+        - '$.properties."spaceborne:swath"'
+      cloudCover:
+        - '{{"query":{{"spaceborne:cloudCover":{{"lte":{cloudCover}}}}}}}'
+        - '$.properties."spaceborne:cloudCover"'
+      sensorMode:
+        - '{{"query":{{"spaceborne:sensorMode":{{"eq":"{sensorMode}"}}}}}}'
+        - '$.properties."spaceborne:sensorMode"'
+      # OpenSearch Parameters for Acquistion Parameters Search (Table 6)
+      startTimeFromAscendingNode:
+        - '{{"query":{{"temporal:endDate":{{"gte":"{startTimeFromAscendingNode#to_iso_utc_datetime}"}}}}}}'
+        - '$.properties."temporal:startDate"'
+      completionTimeFromAscendingNode:
+        - '{{"query":{{"temporal:startDate":{{"lte":"{completionTimeFromAscendingNode#to_iso_utc_datetime}"}}}}}}'
+        - '$.properties."temporal:endDate"'
+      polarizationChannels:
+        - '{{"query":{{"spaceborne:polarization":{{"eq":"{polarizationChannels}"}}}}}}'
+        - '$.properties."spaceborne:polarization"'
+      # Custom parameters (not defined in the base document referenced above)
+      id:
+        - '{{"query":{{"identifier":{{"eq":"{id}"}}}}}}'
+        - '$.properties.identifier'
+      tileIdentifier:
+        - '{{"query":{{"spaceborne:tile":{{"contains":"{tileIdentifier}"}}}}}}'
+        - '{$.properties."spaceborne:tile"#replace_str(r"^T?(.*)$",r"\1")}'
+      geometry:
+        - '{{"intersects":{geometry#to_geojson}}}'
+        - '($.geometry.`str()`.`sub(/^None$/, POLYGON((180 -90, 180 90, -180 90, -180 -90, 180 -90)))`)|($.geometry[*])'
+      providerProductType:
+        - '{{"query":{{"spaceborne:productType":{{"eq":"{providerProductType}"}}}}}}'
+        - '$.null'
+      downloadLink: '$.properties["accessService:endpointURL"]'
+      quicklook: '$.assets[?(@.roles[0] == "overview")].href.`sub(/^(.*)$/, \\1?scope=gdh)`'
+      thumbnail: '$.assets[?(@.roles[0] == "overview")].href.`sub(/^(.*)$/, \\1?scope=gdh)`'
+      # storageStatus set to ONLINE for consistency between providers
+      storageStatus: '{$.null#replace_str("Not Available","ONLINE")}'
+  products:
+    S1_SAR_OCN:
+      providerProductType: OCN
+      productType: PEPS_S1_L2
+      metadata_mapping:
+        cloudCover: '$.null'
+    S1_SAR_GRD:
+      providerProductType: GRD
+      productType: PEPS_S1_L1
+      metadata_mapping:
+        cloudCover: '$.null'
+    S1_SAR_SLC:
+      providerProductType: SLC
+      productType: PEPS_S1_L1
+      metadata_mapping:
+        cloudCover: '$.null'
+    S2_MSI_L1C:
+      productType: PEPS_S2_L1C
+    S2_MSI_L2A_MAJA:
+      productType: MUSCATE_SENTINEL2_SENTINEL2_L2A
+    GENERIC_PRODUCT_TYPE:
+      productType: '{productType}'
+  download: !plugin
+    type: AwsDownload
+    s3_endpoint: https://s3.datalake.cnes.fr
+  auth: !plugin
+    type: AwsAuth
+    auth_error_code: 403
+    matching_conf:
+      s3_endpoint: https://s3.datalake.cnes.fr
+
 ---
 !provider # MARK: cop_ewds
   name: cop_ewds

eodag/resources/stac_api.yml

@@ -799,7 +799,7 @@ components:
         description: |
           Sentinel-2 is a wide-swath, high-resolution, multi-spectral
           imaging mission...
-        license: proprietary
+        license: other
         keywords:
           - copernicus
           - esa
@@ -1416,7 +1416,7 @@ components:
       description: |-
         License(s) of the data as a SPDX
         [License identifier](https://spdx.org/licenses/). Alternatively, use
-        `proprietary` if the license is not on the SPDX license list or
+        `other` if the license is not on the SPDX license list or
         `various` if multiple licenses apply. In these two cases links to the
         license texts SHOULD be added, see the `license` link relation type.
 

eodag/resources/user_conf_template.yml

@@ -157,6 +157,17 @@ geodes:
     download:
         extract:
         output_dir:
+geodes:
+    priority: # Lower value means lower priority (Default: 0)
+    search: # Search parameters configuration
+    auth:
+        credentials:
+            aws_access_key_id:
+            aws_secret_access_key:
+            aws_session_token:
+    download:
+        extract:
+        output_dir:
 hydroweb_next:
     priority: # Lower value means lower priority (Default: 0)
     search: # Search parameters configuration

eodag/rest/cache.py

@@ -16,7 +16,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 import logging
-from typing import Any, Callable, Coroutine, Dict, TypeVar, cast
+from typing import Any, Callable, Coroutine, TypeVar, cast
 
 import orjson
 from cachetools import LRUCache
@@ -48,7 +48,7 @@ async def cached(
     host_cache_key = f"{cache_key}:{host}"
 
     try:
-        c: Dict[str, Any] = request.app.state.cache
+        c: dict[str, Any] = request.app.state.cache
 
         if cached := c.get(host_cache_key):
             logger.debug("Cache result hit")

eodag/rest/config.py

@@ -18,7 +18,7 @@
 from __future__ import annotations
 
 from functools import lru_cache
-from typing import Annotated, List, Union
+from typing import Annotated, Union
 
 from pydantic import Field
 from pydantic.functional_validators import BeforeValidator
@@ -28,7 +28,7 @@ from typing_extensions import Doc
 from eodag.rest.constants import DEFAULT_MAXSIZE, DEFAULT_TTL
 
 
-def str2liststr(raw: Union[str, List[str]]) -> List[str]:
+def str2liststr(raw: Union[str, list[str]]) -> list[str]:
     """Convert str to list[str]"""
     if isinstance(raw, list):
         return raw
@@ -49,7 +49,7 @@ class Settings(BaseSettings):
     stac_api_landing_id: str = "eodag-stac-api"
 
     origin_url_blacklist: Annotated[
-        Union[str, List[str]],
+        Union[str, list[str]],
         BeforeValidator(str2liststr),
         Doc(
             "Hide from clients items assets' alternative URLs starting with URLs from the list"

eodag/rest/core.py

@@ -77,7 +77,7 @@ from eodag.utils.exceptions import (
 )
 
 if TYPE_CHECKING:
-    from typing import Any, Dict, List, Optional, Union
+    from typing import Any, Optional, Union
 
     from fastapi import Request
     from requests.auth import AuthBase
@@ -116,12 +116,12 @@ def get_home_page_content(base_url: str, ipp: Optional[int] = None) -> str:
     reason="Function internally used by get_home_page_content, also deprecated",
     version="2.6.1",
 )
-def format_product_types(product_types: List[Dict[str, Any]]) -> str:
+def format_product_types(product_types: list[dict[str, Any]]) -> str:
     """Format product_types
 
     :param product_types: A list of EODAG product types as returned by the core api
     """
-    result: List[str] = []
+    result: list[str] = []
     for pt in product_types:
         result.append(f'* *__{pt["ID"]}__*: {pt["abstract"]}')
     return "\n".join(sorted(result))
@@ -130,7 +130,7 @@ def format_product_types(product_types: List[Dict[str, Any]]) -> str:
 def search_stac_items(
     request: Request,
     search_request: SearchPostRequest,
-) -> Dict[str, Any]:
+) -> dict[str, Any]:
     """
     Search and retrieve STAC items based on the given search request.
 
@@ -309,8 +309,8 @@ def download_stac_item(
 
 def _order_and_update(
     product: EOProduct,
-    auth: Union[AuthBase, Dict[str, str], None],
-    query_args: Dict[str, Any],
+    auth: Union[AuthBase, dict[str, str], None],
+    query_args: dict[str, Any],
 ) -> None:
     """Order product if needed and update given kwargs with order-status-dict"""
     if product.properties.get("storageStatus") != ONLINE_STATUS and hasattr(
@@ -353,7 +353,7 @@ def _order_and_update(
 
 
 @lru_cache(maxsize=1)
-def get_detailled_collections_list() -> List[Dict[str, Any]]:
+def get_detailled_collections_list() -> list[dict[str, Any]]:
     """Returns detailled collections / product_types list as a list of
     config dicts
 
@@ -370,7 +370,7 @@ async def all_collections(
     instrument: Optional[str] = None,
     constellation: Optional[str] = None,
     datetime: Optional[str] = None,
-) -> Dict[str, Any]:
+) -> dict[str, Any]:
     """Build STAC collections
 
     :param url: Requested URL
@@ -380,7 +380,7 @@ async def all_collections(
     :returns: Collections dictionary
     """
 
-    async def _fetch() -> Dict[str, Any]:
+    async def _fetch() -> dict[str, Any]:
         stac_collection = StacCollection(
             url=request.state.url,
             stac_config=stac_config,
@@ -422,7 +422,7 @@ async def all_collections(
 
 async def get_collection(
     request: Request, collection_id: str, provider: Optional[str] = None
-) -> Dict[str, Any]:
+) -> dict[str, Any]:
     """Build STAC collection by id
 
     :param url: Requested URL
@@ -432,7 +432,7 @@ async def get_collection(
     :returns: Collection dictionary
     """
 
-    async def _fetch() -> Dict[str, Any]:
+    async def _fetch() -> dict[str, Any]:
         stac_collection = StacCollection(
             url=request.state.url,
             stac_config=stac_config,
@@ -455,7 +455,7 @@ async def get_stac_catalogs(
     request: Request,
     url: str,
     provider: Optional[str] = None,
-) -> Dict[str, Any]:
+) -> dict[str, Any]:
     """Build STAC catalog
 
     :param url: Requested URL
@@ -464,7 +464,7 @@ async def get_stac_catalogs(
     :returns: Catalog dictionary
     """
 
-    async def _fetch() -> Dict[str, Any]:
+    async def _fetch() -> dict[str, Any]:
         return StacCatalog(
             url=url,
             stac_config=stac_config,
@@ -523,7 +523,7 @@ def time_interval_overlap(eodag_args: EODAGSearch, catalog: StacCatalog) -> bool
 
 
 @lru_cache(maxsize=1)
-def get_stac_conformance() -> Dict[str, str]:
+def get_stac_conformance() -> dict[str, str]:
     """Build STAC conformance
 
     :returns: conformance dictionary
@@ -540,7 +540,7 @@ def get_stac_api_version() -> str:
 
 
 @lru_cache(maxsize=1)
-def get_stac_extension_oseo(url: str) -> Dict[str, str]:
+def get_stac_extension_oseo(url: str) -> dict[str, str]:
     """Build STAC OGC / OpenSearch Extension for EO
 
     :param url: Requested URL
@@ -571,14 +571,14 @@ async def get_queryables(
     request: Request,
     params: QueryablesGetParams,
     provider: Optional[str] = None,
-) -> Dict[str, Any]:
+) -> dict[str, Any]:
     """Fetch the queryable properties for a collection.
 
     :param collection_id: The ID of the collection.
     :returns: A set containing the STAC standardized queryable properties for a collection.
     """
 
-    async def _fetch() -> Dict[str, Any]:
+    async def _fetch() -> dict[str, Any]:
         python_queryables = eodag_api.list_queryables(
             provider=provider,
             fetch_providers=False,
@@ -589,8 +589,8 @@ async def get_queryables(
             by_alias=True
         )
 
-        properties: Dict[str, Any] = python_queryables_json["properties"]
-        required: List[str] = python_queryables_json.get("required") or []
+        properties: dict[str, Any] = python_queryables_json["properties"]
+        required: list[str] = python_queryables_json.get("required") or []
 
         # productType is either simply removed or replaced by collection later.
         if "productType" in properties:
@@ -598,7 +598,7 @@ async def get_queryables(
         if "productType" in required:
             required.remove("productType")
 
-        stac_properties: Dict[str, Any] = {}
+        stac_properties: dict[str, Any] = {}
 
         # get stac default properties to set prefixes
         stac_item_properties = list(stac_config["item"]["properties"].values())
@@ -687,7 +687,7 @@ def crunch_products(
             f'Unknown crunch name. Use one of: {", ".join(crunchers.keys())}'
         )
 
-    cruncher_config: Dict[str, Any] = {}
+    cruncher_config: dict[str, Any] = {}
     for config_param in cruncher.config_params:
         config_param_value = kwargs.get(config_param)
         if not config_param_value:
@@ -720,13 +720,13 @@ def eodag_api_init() -> None:
             ext_col = StacCollection.ext_stac_collections.get(key)
             if not ext_col:
                 continue
-            platform: Union[str, List[str]] = ext_col.get("summaries", {}).get(
+            platform: Union[str, list[str]] = ext_col.get("summaries", {}).get(
                 "platform"
             )
-            constellation: Union[str, List[str]] = ext_col.get("summaries", {}).get(
+            constellation: Union[str, list[str]] = ext_col.get("summaries", {}).get(
                 "constellation"
             )
-            processing_level: Union[str, List[str]] = ext_col.get("summaries", {}).get(
+            processing_level: Union[str, list[str]] = ext_col.get("summaries", {}).get(
                 "processing:level"
             )
             # Check if platform or constellation are lists and join them into a string if they are

eodag/rest/errors.py

@@ -16,7 +16,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 import logging
-from typing import Dict, List, Tuple, Union
+from typing import Union
 
 from fastapi import FastAPI, Request
 from fastapi.responses import ORJSONResponse
@@ -56,16 +56,16 @@ logger = logging.getLogger("eodag.rest.server")
 class ResponseSearchError(Exception):
     """Represent a EODAG search error response"""
 
-    def __init__(self, errors: List[Tuple[str, Exception]]) -> None:
+    def __init__(self, errors: list[tuple[str, Exception]]) -> None:
         self._errors = errors
 
     @property
-    def errors(self) -> List[Dict[str, Union[str, int]]]:
+    def errors(self) -> list[dict[str, Union[str, int]]]:
         """return errors as a list of dict"""
-        error_list: List[Dict[str, Union[str, int]]] = []
+        error_list: list[dict[str, Union[str, int]]] = []
         for name, exception in self._errors:
 
-            error_dict: Dict[str, Union[str, int]] = {
+            error_dict: dict[str, Union[str, int]] = {
                 "provider": name,
                 "error": exception.__class__.__name__,
             }

eodag/rest/server.py

@@ -23,15 +23,7 @@ import re
 from contextlib import asynccontextmanager
 from importlib.metadata import version
 from json import JSONDecodeError
-from typing import (
-    TYPE_CHECKING,
-    Any,
-    AsyncGenerator,
-    Awaitable,
-    Callable,
-    Dict,
-    Optional,
-)
+from typing import TYPE_CHECKING, Any, AsyncGenerator, Awaitable, Callable, Optional
 
 from fastapi import APIRouter as FastAPIRouter
 from fastapi import FastAPI, HTTPException, Request
@@ -131,7 +123,7 @@ stac_api_config = load_stac_api_config()
     include_in_schema=False,
     status_code=200,
 )
-async def liveness_probe(request: Request) -> Dict[str, bool]:
+async def liveness_probe(request: Request) -> dict[str, bool]:
     "Endpoint meant to be used as liveness probe by deployment platforms"
     return {"success": True}
 
@@ -139,7 +131,7 @@ async def liveness_probe(request: Request) -> Dict[str, bool]:
 @router.api_route(
     methods=["GET", "HEAD"], path="/api", tags=["Capabilities"], include_in_schema=False
 )
-async def eodag_openapi(request: Request) -> Dict[str, Any]:
+async def eodag_openapi(request: Request) -> dict[str, Any]:
     """Customized openapi"""
     logger.debug("URL: /api")
     if app.openapi_schema: