eodag 3.5.1__py3-none-any.whl → 3.7.0__py3-none-any.whl

eodag/api/core.py

@@ -69,6 +69,7 @@ from eodag.utils import (
     DEFAULT_MAX_ITEMS_PER_PAGE,
     DEFAULT_PAGE,
     GENERIC_PRODUCT_TYPE,
+    GENERIC_STAC_PROVIDER,
     HTTP_REQ_TIMEOUT,
     MockResponse,
     _deprecated,
@@ -2006,20 +2007,6 @@ class EODataAccessGateway:
                     nb_res,
                     search_plugin.provider,
                 )
-                # Hitting for instance
-                # https://theia.cnes.fr/atdistrib/resto2/api/collections/SENTINEL2/
-                #   search.json?startDate=2019-03-01&completionDate=2019-06-15
-                #   &processingLevel=LEVEL2A&maxRecords=1&page=1
-                # returns a number (properties.totalResults) that is the number of
-                # products in the collection (here SENTINEL2) instead of the estimated
-                # total number of products matching the search criteria (start/end date).
-                # Remove this warning when this is fixed upstream by THEIA.
-                if search_plugin.provider == "theia":
-                    logger.warning(
-                        "Results found on provider 'theia' is the total number of products "
-                        "available in the searched collection (e.g. SENTINEL2) instead of "
-                        "the total number of products matching the search criteria"
-                    )
         except Exception as e:
             if raise_errors:
                 # Raise the error, letting the application wrapping eodag know that
@@ -2476,3 +2463,31 @@ class EODataAccessGateway:
             )
         # Remove the ID since this is equal to productType.
         plugin.config.product_type_config.pop("ID", None)
+
+    def import_stac_items(self, items_urls: list[str]) -> SearchResult:
+        """Import STAC items from a list of URLs and convert them to SearchResult.
+
+        - Origin provider and download links will be set if item comes from an EODAG
+          server.
+        - If item comes from a known EODAG provider, result will be registered to it,
+          ready to download and its metadata normalized.
+        - If item comes from an unknown provider, a generic STAC provider will be used.
+
+        :param items_urls: A list of STAC items URLs to import
+        :returns: A SearchResult containing the imported STAC items
+        """
+        json_items = []
+        for item_url in items_urls:
+            json_items.extend(fetch_stac_items(item_url))
+
+        # add a generic STAC provider that might be needed to handle the items
+        self.add_provider(GENERIC_STAC_PROVIDER)
+
+        results = SearchResult([])
+        for json_item in json_items:
+            if search_result := SearchResult._from_stac_item(
+                json_item, self._plugins_manager
+            ):
+                results.extend(search_result)
+
+        return results

eodag/api/product/__init__.py

@@ -17,6 +17,12 @@
 # limitations under the License.
 #
 """EODAG product package"""
+
+from typing import TYPE_CHECKING, Any, Optional
+
+if TYPE_CHECKING:
+    from eodag.plugins.manager import PluginManager
+
 try:
     # import from eodag-cube if installed
     from eodag_cube.api.product import (  # pyright: ignore[reportMissingImports]
@@ -30,3 +36,27 @@ except ImportError:
 
 # exportable content
 __all__ = ["Asset", "AssetsDict", "EOProduct"]
+
+
+def unregistered_product_from_item(
+    feature: dict[str, Any], provider: str, plugins_manager: "PluginManager"
+) -> Optional[EOProduct]:
+    """Create an EOProduct from a STAC item, map its metadata, but without registering its plugins.
+
+    :param feature: The STAC item to convert into an EOProduct.
+    :param provider: The associated provider from which configuration should be used for mapping.
+    :param plugins_manager: The plugins manager instance to use for retrieving search plugins.
+    :returns: An EOProduct instance if the item can be normalized, otherwise None.
+    """
+    for search_plugin in plugins_manager.get_search_plugins(provider=provider):
+        if hasattr(search_plugin, "normalize_results"):
+            products = search_plugin.normalize_results([feature])
+            if len(products) > 0:
+                # properties cleanup
+                for prop in ("start_datetime", "end_datetime"):
+                    products[0].properties.pop(prop, None)
+                # set product type if not already set
+                if products[0].product_type is None:
+                    products[0].product_type = products[0].properties.get("productType")
+                return products[0]
+    return None

eodag/api/product/metadata_mapping.py

@@ -219,7 +219,7 @@ def format_metadata(search_param: str, *args: Any, **kwargs: Any) -> str:
                 elif value is not None:
                     converted = self.custom_converter(value)
                 else:
-                    converted = ""
+                    converted = None
                 # Clear this state variable in case the same converter is used to
                 # resolve other named arguments
                 self.custom_converter = None
@@ -374,6 +374,18 @@ def format_metadata(search_param: str, *args: Any, **kwargs: Any) -> str:
         def convert_to_geojson(value: Any) -> str:
             return geojson.dumps(value)
 
+        @staticmethod
+        def convert_to_geojson_polytope(
+            value: BaseGeometry,
+        ) -> Union[dict[Any, Any], str]:
+            # ECMWF Polytope uses non-geojson structure for features
+            if isinstance(value, Polygon):
+                return {
+                    "type": "polygon",
+                    "shape": [[y, x] for x, y in value.exterior.coords],
+                }
+            raise ValidationError("to_geojson_polytope only accepts shapely Polygon")
+
         @staticmethod
         def convert_from_ewkt(ewkt_string: str) -> Union[BaseGeometry, str]:
             """Convert EWKT (Extended Well-Known text) to shapely geometry"""
@@ -488,10 +500,14 @@ def format_metadata(search_param: str, *args: Any, **kwargs: Any) -> str:
 
         @staticmethod
         def convert_get_group_name(string: str, pattern: str) -> str:
+            sanitized_pattern = pattern.replace(" ", "_SPACE_")
             try:
-                match = re.search(pattern, str(string))
+                match = re.search(sanitized_pattern, str(string))
                 if match:
-                    return match.lastgroup or NOT_AVAILABLE
+                    if result := match.lastgroup:
+                        return result.replace("_SPACE_", " ")
+                    else:
+                        return NOT_AVAILABLE
             except AttributeError:
                 pass
             logger.warning(
@@ -1342,6 +1358,7 @@ def format_query_params(
                     formatted_query_param = remove_str_array_quotes(
                         formatted_query_param
                     )
+
                     # json query string (for POST request)
                     update_nested_dict(
                         query_params,

eodag/api/search_result.py

@@ -17,23 +17,30 @@
 # limitations under the License.
 from __future__ import annotations
 
+import logging
 from collections import UserList
 from typing import TYPE_CHECKING, Annotated, Any, Iterable, Optional, Union
 
 from shapely.geometry import GeometryCollection, shape
 from typing_extensions import Doc
 
-from eodag.api.product import EOProduct
+from eodag.api.product import EOProduct, unregistered_product_from_item
 from eodag.plugins.crunch.filter_date import FilterDate
 from eodag.plugins.crunch.filter_latest_intersect import FilterLatestIntersect
 from eodag.plugins.crunch.filter_latest_tpl_name import FilterLatestByName
 from eodag.plugins.crunch.filter_overlap import FilterOverlap
 from eodag.plugins.crunch.filter_property import FilterProperty
+from eodag.utils import GENERIC_STAC_PROVIDER, STAC_SEARCH_PLUGINS
+from eodag.utils.exceptions import MisconfiguredError
 
 if TYPE_CHECKING:
     from shapely.geometry.base import BaseGeometry
 
     from eodag.plugins.crunch.base import Crunch
+    from eodag.plugins.manager import PluginManager
+
+
+logger = logging.getLogger("eodag.search_result")
 
 
 class SearchResult(UserList[EOProduct]):
@@ -211,6 +218,153 @@ class SearchResult(UserList[EOProduct]):
 
         return super().extend(other)
 
+    @classmethod
+    def _from_stac_item(
+        cls, feature: dict[str, Any], plugins_manager: PluginManager
+    ) -> SearchResult:
+        """Create a SearchResult from a STAC item.
+
+        :param feature: A STAC item as a dictionary
+        :param plugins_manager: The EODAG plugin manager instance
+        :returns: A SearchResult containing the EOProduct(s) created from the STAC item
+        """
+        # Try importing from EODAG Server
+        if results := _import_stac_item_from_eodag_server(feature, plugins_manager):
+            return results
+
+        # try importing from a known STAC provider
+        if results := _import_stac_item_from_known_provider(feature, plugins_manager):
+            return results
+
+        # try importing from an unknown STAC provider
+        return _import_stac_item_from_unknown_provider(feature, plugins_manager)
+
+
+def _import_stac_item_from_eodag_server(
+    feature: dict[str, Any], plugins_manager: PluginManager
+) -> Optional[SearchResult]:
+    """Import a STAC item from EODAG Server.
+
+    :param feature: A STAC item as a dictionary
+    :param plugins_manager: The EODAG plugin manager instance
+    :returns: A SearchResult containing the EOProduct(s) created from the STAC item
+    """
+    provider = None
+    if backends := feature["properties"].get("federation:backends"):
+        provider = backends[0]
+    elif providers := feature["properties"].get("providers"):
+        provider = providers[0].get("name")
+    if provider is not None:
+        logger.debug("Trying to import STAC item from EODAG Server")
+        # assets coming from a STAC provider
+        assets = {
+            k: v["alternate"]["origin"]
+            for k, v in feature.get("assets", {}).items()
+            if k not in ("thumbnail", "downloadLink")
+            and "origin" in v.get("alternate", {})
+        }
+        if assets:
+            updated_item = {**feature, **{"assets": assets}}
+        else:
+            # item coming from a non-STAC provider
+            updated_item = {**feature}
+            download_link = (
+                feature.get("assets", {})
+                .get("downloadLink", {})
+                .get("alternate", {})
+                .get("origin", {})
+                .get("href")
+            )
+            if download_link:
+                updated_item["assets"] = {}
+                updated_item["links"] = [{"rel": "self", "href": download_link}]
+            else:
+                updated_item = {}
+        try:
+            eo_product = unregistered_product_from_item(
+                updated_item, GENERIC_STAC_PROVIDER, plugins_manager
+            )
+        except MisconfiguredError:
+            eo_product = None
+        if eo_product is not None:
+            eo_product.provider = provider
+            eo_product._register_downloader_from_manager(plugins_manager)
+            return SearchResult([eo_product])
+    return None
+
+
+def _import_stac_item_from_known_provider(
+    feature: dict[str, Any], plugins_manager: PluginManager
+) -> Optional[SearchResult]:
+    """Import a STAC item from an already-configured STAC provider.
+
+    :param feature: A STAC item as a dictionary
+    :param plugins_manager: The EODAG plugin manager instance
+    :returns: A SearchResult containing the EOProduct(s) created from the STAC item
+    """
+    item_hrefs = [f for f in feature.get("links", []) if f.get("rel") == "self"]
+    item_href = item_hrefs[0]["href"] if len(item_hrefs) > 0 else None
+    imported_products = SearchResult([])
+    for search_plugin in plugins_manager.get_search_plugins():
+        # only try STAC search plugins
+        if (
+            search_plugin.config.type in STAC_SEARCH_PLUGINS
+            and search_plugin.provider != GENERIC_STAC_PROVIDER
+            and hasattr(search_plugin, "normalize_results")
+        ):
+            provider_base_url = search_plugin.config.api_endpoint.removesuffix(
+                "/search"
+            )
+            # compare the item href with the provider base URL
+            if item_href and item_href.startswith(provider_base_url):
+                products = search_plugin.normalize_results([feature])
+                if len(products) == 0 or len(products[0].assets) == 0:
+                    continue
+                logger.debug(
+                    "Trying to import STAC item from %s", search_plugin.provider
+                )
+                eo_product = products[0]
+
+                configured_pts = [
+                    k
+                    for k, v in search_plugin.config.products.items()
+                    if v.get("productType") == feature.get("collection")
+                ]
+                if len(configured_pts) > 0:
+                    eo_product.product_type = configured_pts[0]
+                else:
+                    eo_product.product_type = feature.get("collection")
+
+                eo_product._register_downloader_from_manager(plugins_manager)
+                imported_products.append(eo_product)
+    if len(imported_products) > 0:
+        return imported_products
+    return None
+
+
+def _import_stac_item_from_unknown_provider(
+    feature: dict[str, Any], plugins_manager: PluginManager
+) -> SearchResult:
+    """Import a STAC item from an unknown STAC provider.
+
+    :param feature: A STAC item as a dictionary
+    :param plugins_manager: The EODAG plugin manager instance
+    :returns: A SearchResult containing the EOProduct(s) created from the STAC item
+    """
+    try:
+        logger.debug("Trying to import STAC item from unknown provider")
+        eo_product = unregistered_product_from_item(
+            feature, GENERIC_STAC_PROVIDER, plugins_manager
+        )
+    except MisconfiguredError:
+        pass
+    if eo_product is not None:
+        eo_product.product_type = feature.get("collection")
+        eo_product._register_downloader_from_manager(plugins_manager)
+        return SearchResult([eo_product])
+    else:
+        return SearchResult([])
+
 
 class RawSearchResult(UserList[dict[str, Any]]):
     """An object representing a collection of raw/unparsed search results obtained from a provider.

eodag/cli.py

@@ -49,11 +49,12 @@ import sys
 import textwrap
 from importlib.metadata import metadata
 from typing import TYPE_CHECKING, Any, Mapping
+from urllib.parse import parse_qs
 
 import click
 
-from eodag.api.core import EODataAccessGateway
-from eodag.utils import DEFAULT_ITEMS_PER_PAGE, DEFAULT_PAGE, parse_qs
+from eodag.api.core import EODataAccessGateway, SearchResult
+from eodag.utils import DEFAULT_ITEMS_PER_PAGE, DEFAULT_PAGE
 from eodag.utils.exceptions import NoMatchingProductType, UnsupportedProvider
 from eodag.utils.logging import setup_logging
 
@@ -109,14 +110,15 @@ class MutuallyExclusiveOption(click.Option):
         """Raise error or use parent handle_parse_result()"""
         if self.mutually_exclusive.intersection(opts) and self.name in opts:
             raise click.UsageError(
-                "Illegal usage: `{}` is mutually exclusive with "
-                "arguments `{}`.".format(self.name, ", ".join(self.mutually_exclusive))
+                "Illegal usage: `{}` is mutually exclusive with arguments `{}`.".format(
+                    self.name, ", ".join(self.mutually_exclusive)
+                )
             )
 
         return super(MutuallyExclusiveOption, self).handle_parse_result(ctx, opts, args)
 
 
-@click.group()
+@click.group(chain=True)
 @click.option(
     "-v",
     "--verbose",
@@ -212,6 +214,18 @@ def version() -> None:
     "-S", "--sensorType", help="Search for products matching this type of sensor"
 )
 @click.option("--id", help="Search for the product identified by this id")
+@click.option(
+    "--locations",
+    type=str,
+    help="Custom query-string argument(s) to select locations. "
+    "Format :'key1=value1&key2=value2'. Example: --locations country=FRA&continent=Africa",
+)
+@click.option(
+    "-q",
+    "--query",
+    type=str,
+    help="Custom query-string argument(s). Format :'key1=value1&key2=value2'",
+)
 @click.option(
     "--cruncher",
     type=click.Choice(CRUNCHERS),
@@ -262,19 +276,9 @@ def version() -> None:
 @click.option(
     "--count",
     is_flag=True,
-    help="Whether to run a query with a count request or not.",
-)
-@click.option(
-    "--locations",
-    type=str,
-    help="Custom query-string argument(s) to select locations. "
-    "Format :'key1=value1&key2=value2'. Example: --locations country=FRA&continent=Africa",
-)
-@click.option(
-    "-q",
-    "--query",
-    type=str,
-    help="Custom query-string argument(s). Format :'key1=value1&key2=value2'",
+    help="Make a count request together with search (Enabling count will significantly "
+    "slow down search requests for some providers, and might be unavailable for some"
+    "others).",
 )
 @click.pass_context
 def search_crunch(ctx: Context, **kwargs: Any) -> None:
@@ -407,6 +411,7 @@ def search_crunch(ctx: Context, **kwargs: Any) -> None:
         storage_filepath += ".geojson"
     result_storage = gateway.serialize(results, filename=storage_filepath)
     click.echo("Results stored at '{}'".format(result_storage))
+    ctx.obj["search_results"] = results
 
 
 @eodag.command(name="list", help="List supported product types")
@@ -528,12 +533,26 @@ def discover_pt(ctx: Context, **kwargs: Any) -> None:
     click.echo("Results stored at '{}'".format(storage_filepath))
 
 
-@eodag.command(help="Download a list of products from a serialized search result")
+@eodag.command(
+    help="""Download a list of products from a serialized search result or STAC items URLs/paths
+
+Examples:
+
+  eodag download --search-results /path/to/search_results.geojson
+
+  eodag download --stac-item https://example.com/stac/item1.json --stac-item /path/to/item2.json
+""",
+)
 @click.option(
     "--search-results",
     type=click.Path(exists=True, dir_okay=False),
     help="Path to a serialized search result",
 )
+@click.option(
+    "--stac-item",
+    multiple=True,
+    help="URL/path of a STAC item to download (multiple values accepted)",
+)
 @click.option(
     "-f",
     "--conf",
@@ -546,13 +565,20 @@ def discover_pt(ctx: Context, **kwargs: Any) -> None:
     show_default=False,
     help="Download only quicklooks of products instead full set of files",
 )
+@click.option(
+    "--output-dir",
+    type=click.Path(dir_okay=True, file_okay=False),
+    help="Products or quicklooks download directory (Default: local temporary directory)",
+)
 @click.pass_context
 def download(ctx: Context, **kwargs: Any) -> None:
     """Download a bunch of products from a serialized search result"""
     search_result_path = kwargs.pop("search_results")
-    if not search_result_path:
+    stac_items = kwargs.pop("stac_item")
+    search_results = ctx.obj.get("search_results")
+    if not search_result_path and not stac_items and search_results is None:
         with click.Context(download) as ctx:
-            click.echo("Nothing to do (no search results file provided)")
+            click.echo("Nothing to do (no search results file or stac item provided)")
             click.echo(download.get_help(ctx))
         sys.exit(1)
     setup_logging(verbose=ctx.obj["verbosity"])
@@ -561,25 +587,24 @@ def download(ctx: Context, **kwargs: Any) -> None:
         conf_file = click.format_filename(conf_file)
 
     satim_api = EODataAccessGateway(user_conf_file_path=conf_file)
-    search_results = satim_api.deserialize(search_result_path)
 
+    search_results = search_results or SearchResult([])
+    if search_result_path:
+        search_results.extend(satim_api.deserialize_and_register(search_result_path))
+    if stac_items:
+        search_results.extend(satim_api.import_stac_items(list(stac_items)))
+
+    output_dir = kwargs.pop("output_dir")
     get_quicklooks = kwargs.pop("quicklooks")
+
     if get_quicklooks:
+        # Download only quicklooks
         click.echo(
             "Flag 'quicklooks' specified, downloading only quicklooks of products"
         )
 
         for idx, product in enumerate(search_results):
-            if product.downloader is None:
-                downloader = satim_api._plugins_manager.get_download_plugin(product)
-                auth = product.downloader_auth
-                if auth is None:
-                    auth = satim_api._plugins_manager.get_auth_plugin(
-                        downloader, product
-                    )
-                search_results[idx].register_downloader(downloader, auth)
-
-            downloaded_file = product.get_quicklook()
+            downloaded_file = product.get_quicklook(output_dir=output_dir)
             if not downloaded_file:
                 click.echo(
                     "A quicklook may have been downloaded but we cannot locate it. "
@@ -589,18 +614,8 @@ def download(ctx: Context, **kwargs: Any) -> None:
                 click.echo("Downloaded {}".format(downloaded_file))
 
     else:
-        # register downloader
-        for idx, product in enumerate(search_results):
-            if product.downloader is None:
-                downloader = satim_api._plugins_manager.get_download_plugin(product)
-                auth = product.downloader_auth
-                if auth is None:
-                    auth = satim_api._plugins_manager.get_auth_plugin(
-                        downloader, product
-                    )
-                search_results[idx].register_downloader(downloader, auth)
-
-        downloaded_files = satim_api.download_all(search_results)
+        # Download products
+        downloaded_files = satim_api.download_all(search_results, output_dir=output_dir)
         if downloaded_files and len(downloaded_files) > 0:
             for downloaded_file in downloaded_files:
                 if downloaded_file is None:
@@ -674,6 +689,7 @@ def serve_rest(
     setup_logging(verbose=ctx.obj["verbosity"])
     try:
         import uvicorn
+        import uvicorn.config
     except ImportError:
         raise ImportError(
             "Feature not available, please install eodag[server] or eodag[all]"

eodag/config.py

@@ -46,6 +46,7 @@ from jsonpath_ng import JSONPath
 from eodag.api.product.metadata_mapping import mtd_cfg_as_conversion_and_querypath
 from eodag.utils import (
     HTTP_REQ_TIMEOUT,
+    STAC_SEARCH_PLUGINS,
     USER_AGENT,
     cached_yaml_load,
     cached_yaml_load_all,
@@ -451,6 +452,11 @@ class PluginConfig(yaml.YAMLObject):
     discover_queryables: PluginConfig.DiscoverQueryables
     #: :class:`~eodag.plugins.search.base.Search` The mapping between eodag metadata and the plugin specific metadata
     metadata_mapping: dict[str, Union[str, list[str]]]
+    #: :class:`~eodag.plugins.search.base.Search` :attr:`~eodag.config.PluginConfig.metadata_mapping` got from the given
+    #: product type
+    metadata_mapping_from_product: str
+    #: :class:`~eodag.plugins.search.base.Search` A mapping for the metadata of individual assets
+    assets_mapping: dict[str, dict[str, Any]]
     #: :class:`~eodag.plugins.search.base.Search` Parameters to remove from queryables
     remove_from_queryables: list[str]
     #: :class:`~eodag.plugins.search.base.Search` Parameters to be passed as is in the search url query string
@@ -819,12 +825,7 @@ def provider_config_init(
         if (
             stac_search_default_conf is not None
             and provider_config.search
-            and provider_config.search.type
-            in [
-                "StacSearch",
-                "StacListAssets",
-                "StaticStacSearch",
-            ]
+            and provider_config.search.type in STAC_SEARCH_PLUGINS
         ):
             # search config set to stac defaults overriden with provider config
             per_provider_stac_provider_config = deepcopy(stac_search_default_conf)

eodag/plugins/authentication/openid_connect.py

@@ -23,6 +23,7 @@ import string
 from datetime import datetime, timedelta, timezone
 from random import SystemRandom
 from typing import TYPE_CHECKING, Any, Optional
+from urllib.parse import parse_qs, urlparse
 
 import jwt
 import requests
@@ -34,9 +35,7 @@ from eodag.utils import (
     DEFAULT_TOKEN_EXPIRATION_MARGIN,
     HTTP_REQ_TIMEOUT,
     USER_AGENT,
-    parse_qs,
     repeatfunc,
-    urlparse,
 )
 from eodag.utils.exceptions import (
     AuthenticationError,