eodag 3.5.1__py3-none-any.whl → 3.6.0__py3-none-any.whl

eodag/api/core.py

@@ -69,6 +69,7 @@ from eodag.utils import (
     DEFAULT_MAX_ITEMS_PER_PAGE,
     DEFAULT_PAGE,
     GENERIC_PRODUCT_TYPE,
+    GENERIC_STAC_PROVIDER,
     HTTP_REQ_TIMEOUT,
     MockResponse,
     _deprecated,
@@ -2476,3 +2477,31 @@ class EODataAccessGateway:
             )
         # Remove the ID since this is equal to productType.
         plugin.config.product_type_config.pop("ID", None)
+
+    def import_stac_items(self, items_urls: list[str]) -> SearchResult:
+        """Import STAC items from a list of URLs and convert them to SearchResult.
+
+        - Origin provider and download links will be set if item comes from an EODAG
+          server.
+        - If item comes from a known EODAG provider, result will be registered to it,
+          ready to download and its metadata normalized.
+        - If item comes from an unknown provider, a generic STAC provider will be used.
+
+        :param items_urls: A list of STAC items URLs to import
+        :returns: A SearchResult containing the imported STAC items
+        """
+        json_items = []
+        for item_url in items_urls:
+            json_items.extend(fetch_stac_items(item_url))
+
+        # add a generic STAC provider that might be needed to handle the items
+        self.add_provider(GENERIC_STAC_PROVIDER)
+
+        results = SearchResult([])
+        for json_item in json_items:
+            if search_result := SearchResult._from_stac_item(
+                json_item, self._plugins_manager
+            ):
+                results.extend(search_result)
+
+        return results

eodag/api/product/__init__.py

@@ -17,6 +17,12 @@
 # limitations under the License.
 #
 """EODAG product package"""
+
+from typing import TYPE_CHECKING, Any, Optional
+
+if TYPE_CHECKING:
+    from eodag.plugins.manager import PluginManager
+
 try:
     # import from eodag-cube if installed
     from eodag_cube.api.product import (  # pyright: ignore[reportMissingImports]
@@ -30,3 +36,27 @@ except ImportError:
 
 # exportable content
 __all__ = ["Asset", "AssetsDict", "EOProduct"]
+
+
+def unregistered_product_from_item(
+    feature: dict[str, Any], provider: str, plugins_manager: "PluginManager"
+) -> Optional[EOProduct]:
+    """Create an EOProduct from a STAC item, map its metadata, but without registering its plugins.
+
+    :param feature: The STAC item to convert into an EOProduct.
+    :param provider: The associated provider from which configuration should be used for mapping.
+    :param plugins_manager: The plugins manager instance to use for retrieving search plugins.
+    :returns: An EOProduct instance if the item can be normalized, otherwise None.
+    """
+    for search_plugin in plugins_manager.get_search_plugins(provider=provider):
+        if hasattr(search_plugin, "normalize_results"):
+            products = search_plugin.normalize_results([feature])
+            if len(products) > 0:
+                # properties cleanup
+                for prop in ("start_datetime", "end_datetime"):
+                    products[0].properties.pop(prop, None)
+                # set product type if not already set
+                if products[0].product_type is None:
+                    products[0].product_type = products[0].properties.get("productType")
+                return products[0]
+    return None

eodag/api/search_result.py

@@ -17,23 +17,30 @@
 # limitations under the License.
 from __future__ import annotations
 
+import logging
 from collections import UserList
 from typing import TYPE_CHECKING, Annotated, Any, Iterable, Optional, Union
 
 from shapely.geometry import GeometryCollection, shape
 from typing_extensions import Doc
 
-from eodag.api.product import EOProduct
+from eodag.api.product import EOProduct, unregistered_product_from_item
 from eodag.plugins.crunch.filter_date import FilterDate
 from eodag.plugins.crunch.filter_latest_intersect import FilterLatestIntersect
 from eodag.plugins.crunch.filter_latest_tpl_name import FilterLatestByName
 from eodag.plugins.crunch.filter_overlap import FilterOverlap
 from eodag.plugins.crunch.filter_property import FilterProperty
+from eodag.utils import GENERIC_STAC_PROVIDER, STAC_SEARCH_PLUGINS
+from eodag.utils.exceptions import MisconfiguredError
 
 if TYPE_CHECKING:
     from shapely.geometry.base import BaseGeometry
 
     from eodag.plugins.crunch.base import Crunch
+    from eodag.plugins.manager import PluginManager
+
+
+logger = logging.getLogger("eodag.search_result")
 
 
 class SearchResult(UserList[EOProduct]):
@@ -211,6 +218,153 @@ class SearchResult(UserList[EOProduct]):
 
         return super().extend(other)
 
+    @classmethod
+    def _from_stac_item(
+        cls, feature: dict[str, Any], plugins_manager: PluginManager
+    ) -> SearchResult:
+        """Create a SearchResult from a STAC item.
+
+        :param feature: A STAC item as a dictionary
+        :param plugins_manager: The EODAG plugin manager instance
+        :returns: A SearchResult containing the EOProduct(s) created from the STAC item
+        """
+        # Try importing from EODAG Server
+        if results := _import_stac_item_from_eodag_server(feature, plugins_manager):
+            return results
+
+        # try importing from a known STAC provider
+        if results := _import_stac_item_from_known_provider(feature, plugins_manager):
+            return results
+
+        # try importing from an unknown STAC provider
+        return _import_stac_item_from_unknown_provider(feature, plugins_manager)
+
+
+def _import_stac_item_from_eodag_server(
+    feature: dict[str, Any], plugins_manager: PluginManager
+) -> Optional[SearchResult]:
+    """Import a STAC item from EODAG Server.
+
+    :param feature: A STAC item as a dictionary
+    :param plugins_manager: The EODAG plugin manager instance
+    :returns: A SearchResult containing the EOProduct(s) created from the STAC item
+    """
+    provider = None
+    if backends := feature["properties"].get("federation:backends"):
+        provider = backends[0]
+    elif providers := feature["properties"].get("providers"):
+        provider = providers[0].get("name")
+    if provider is not None:
+        logger.debug("Trying to import STAC item from EODAG Server")
+        # assets coming from a STAC provider
+        assets = {
+            k: v["alternate"]["origin"]
+            for k, v in feature.get("assets", {}).items()
+            if k not in ("thumbnail", "downloadLink")
+            and "origin" in v.get("alternate", {})
+        }
+        if assets:
+            updated_item = {**feature, **{"assets": assets}}
+        else:
+            # item coming from a non-STAC provider
+            updated_item = {**feature}
+            download_link = (
+                feature.get("assets", {})
+                .get("downloadLink", {})
+                .get("alternate", {})
+                .get("origin", {})
+                .get("href")
+            )
+            if download_link:
+                updated_item["assets"] = {}
+                updated_item["links"] = [{"rel": "self", "href": download_link}]
+            else:
+                updated_item = {}
+        try:
+            eo_product = unregistered_product_from_item(
+                updated_item, GENERIC_STAC_PROVIDER, plugins_manager
+            )
+        except MisconfiguredError:
+            eo_product = None
+        if eo_product is not None:
+            eo_product.provider = provider
+            eo_product._register_downloader_from_manager(plugins_manager)
+            return SearchResult([eo_product])
+    return None
+
+
+def _import_stac_item_from_known_provider(
+    feature: dict[str, Any], plugins_manager: PluginManager
+) -> Optional[SearchResult]:
+    """Import a STAC item from an already-configured STAC provider.
+
+    :param feature: A STAC item as a dictionary
+    :param plugins_manager: The EODAG plugin manager instance
+    :returns: A SearchResult containing the EOProduct(s) created from the STAC item
+    """
+    item_hrefs = [f for f in feature.get("links", []) if f.get("rel") == "self"]
+    item_href = item_hrefs[0]["href"] if len(item_hrefs) > 0 else None
+    imported_products = SearchResult([])
+    for search_plugin in plugins_manager.get_search_plugins():
+        # only try STAC search plugins
+        if (
+            search_plugin.config.type in STAC_SEARCH_PLUGINS
+            and search_plugin.provider != GENERIC_STAC_PROVIDER
+            and hasattr(search_plugin, "normalize_results")
+        ):
+            provider_base_url = search_plugin.config.api_endpoint.removesuffix(
+                "/search"
+            )
+            # compare the item href with the provider base URL
+            if item_href and item_href.startswith(provider_base_url):
+                products = search_plugin.normalize_results([feature])
+                if len(products) == 0 or len(products[0].assets) == 0:
+                    continue
+                logger.debug(
+                    "Trying to import STAC item from %s", search_plugin.provider
+                )
+                eo_product = products[0]
+
+                configured_pts = [
+                    k
+                    for k, v in search_plugin.config.products.items()
+                    if v.get("productType") == feature.get("collection")
+                ]
+                if len(configured_pts) > 0:
+                    eo_product.product_type = configured_pts[0]
+                else:
+                    eo_product.product_type = feature.get("collection")
+
+                eo_product._register_downloader_from_manager(plugins_manager)
+                imported_products.append(eo_product)
+    if len(imported_products) > 0:
+        return imported_products
+    return None
+
+
+def _import_stac_item_from_unknown_provider(
+    feature: dict[str, Any], plugins_manager: PluginManager
+) -> SearchResult:
+    """Import a STAC item from an unknown STAC provider.
+
+    :param feature: A STAC item as a dictionary
+    :param plugins_manager: The EODAG plugin manager instance
+    :returns: A SearchResult containing the EOProduct(s) created from the STAC item
+    """
+    try:
+        logger.debug("Trying to import STAC item from unknown provider")
+        eo_product = unregistered_product_from_item(
+            feature, GENERIC_STAC_PROVIDER, plugins_manager
+        )
+    except MisconfiguredError:
+        pass
+    if eo_product is not None:
+        eo_product.product_type = feature.get("collection")
+        eo_product._register_downloader_from_manager(plugins_manager)
+        return SearchResult([eo_product])
+    else:
+        return SearchResult([])
+
 
 class RawSearchResult(UserList[dict[str, Any]]):
     """An object representing a collection of raw/unparsed search results obtained from a provider.

eodag/cli.py

@@ -52,7 +52,7 @@ from typing import TYPE_CHECKING, Any, Mapping
 
 import click
 
-from eodag.api.core import EODataAccessGateway
+from eodag.api.core import EODataAccessGateway, SearchResult
 from eodag.utils import DEFAULT_ITEMS_PER_PAGE, DEFAULT_PAGE, parse_qs
 from eodag.utils.exceptions import NoMatchingProductType, UnsupportedProvider
 from eodag.utils.logging import setup_logging
@@ -116,7 +116,7 @@ class MutuallyExclusiveOption(click.Option):
         return super(MutuallyExclusiveOption, self).handle_parse_result(ctx, opts, args)
 
 
-@click.group()
+@click.group(chain=True)
 @click.option(
     "-v",
     "--verbose",
@@ -212,6 +212,18 @@ def version() -> None:
     "-S", "--sensorType", help="Search for products matching this type of sensor"
 )
 @click.option("--id", help="Search for the product identified by this id")
+@click.option(
+    "--locations",
+    type=str,
+    help="Custom query-string argument(s) to select locations. "
+    "Format :'key1=value1&key2=value2'. Example: --locations country=FRA&continent=Africa",
+)
+@click.option(
+    "-q",
+    "--query",
+    type=str,
+    help="Custom query-string argument(s). Format :'key1=value1&key2=value2'",
+)
 @click.option(
     "--cruncher",
     type=click.Choice(CRUNCHERS),
@@ -262,19 +274,9 @@ def version() -> None:
 @click.option(
     "--count",
     is_flag=True,
-    help="Whether to run a query with a count request or not.",
-)
-@click.option(
-    "--locations",
-    type=str,
-    help="Custom query-string argument(s) to select locations. "
-    "Format :'key1=value1&key2=value2'. Example: --locations country=FRA&continent=Africa",
-)
-@click.option(
-    "-q",
-    "--query",
-    type=str,
-    help="Custom query-string argument(s). Format :'key1=value1&key2=value2'",
+    help="Make a count request together with search (Enabling count will significantly "
+    "slow down search requests for some providers, and might be unavailable for some"
+    "others).",
 )
 @click.pass_context
 def search_crunch(ctx: Context, **kwargs: Any) -> None:
@@ -407,6 +409,7 @@ def search_crunch(ctx: Context, **kwargs: Any) -> None:
         storage_filepath += ".geojson"
     result_storage = gateway.serialize(results, filename=storage_filepath)
     click.echo("Results stored at '{}'".format(result_storage))
+    ctx.obj["search_results"] = results
 
 
 @eodag.command(name="list", help="List supported product types")
@@ -528,12 +531,26 @@ def discover_pt(ctx: Context, **kwargs: Any) -> None:
     click.echo("Results stored at '{}'".format(storage_filepath))
 
 
-@eodag.command(help="Download a list of products from a serialized search result")
+@eodag.command(
+    help="""Download a list of products from a serialized search result or STAC items URLs/paths
+
+Examples:
+
+  eodag download --search-results /path/to/search_results.geojson
+
+  eodag download --stac-item https://example.com/stac/item1.json --stac-item /path/to/item2.json
+""",
+)
 @click.option(
     "--search-results",
     type=click.Path(exists=True, dir_okay=False),
     help="Path to a serialized search result",
 )
+@click.option(
+    "--stac-item",
+    multiple=True,
+    help="URL/path of a STAC item to download (multiple values accepted)",
+)
 @click.option(
     "-f",
     "--conf",
@@ -546,13 +563,20 @@ def discover_pt(ctx: Context, **kwargs: Any) -> None:
     show_default=False,
     help="Download only quicklooks of products instead full set of files",
 )
+@click.option(
+    "--output-dir",
+    type=click.Path(dir_okay=True, file_okay=False),
+    help="Products or quicklooks download directory (Default: local temporary directory)",
+)
 @click.pass_context
 def download(ctx: Context, **kwargs: Any) -> None:
     """Download a bunch of products from a serialized search result"""
     search_result_path = kwargs.pop("search_results")
-    if not search_result_path:
+    stac_items = kwargs.pop("stac_item")
+    search_results = ctx.obj.get("search_results")
+    if not search_result_path and not stac_items and search_results is None:
         with click.Context(download) as ctx:
-            click.echo("Nothing to do (no search results file provided)")
+            click.echo("Nothing to do (no search results file or stac item provided)")
             click.echo(download.get_help(ctx))
         sys.exit(1)
     setup_logging(verbose=ctx.obj["verbosity"])
@@ -561,25 +585,24 @@ def download(ctx: Context, **kwargs: Any) -> None:
         conf_file = click.format_filename(conf_file)
 
     satim_api = EODataAccessGateway(user_conf_file_path=conf_file)
-    search_results = satim_api.deserialize(search_result_path)
 
+    search_results = search_results or SearchResult([])
+    if search_result_path:
+        search_results.extend(satim_api.deserialize_and_register(search_result_path))
+    if stac_items:
+        search_results.extend(satim_api.import_stac_items(list(stac_items)))
+
+    output_dir = kwargs.pop("output_dir")
     get_quicklooks = kwargs.pop("quicklooks")
+
     if get_quicklooks:
+        # Download only quicklooks
         click.echo(
             "Flag 'quicklooks' specified, downloading only quicklooks of products"
         )
 
         for idx, product in enumerate(search_results):
-            if product.downloader is None:
-                downloader = satim_api._plugins_manager.get_download_plugin(product)
-                auth = product.downloader_auth
-                if auth is None:
-                    auth = satim_api._plugins_manager.get_auth_plugin(
-                        downloader, product
-                    )
-                search_results[idx].register_downloader(downloader, auth)
-
-            downloaded_file = product.get_quicklook()
+            downloaded_file = product.get_quicklook(output_dir=output_dir)
             if not downloaded_file:
                 click.echo(
                     "A quicklook may have been downloaded but we cannot locate it. "
@@ -589,18 +612,8 @@ def download(ctx: Context, **kwargs: Any) -> None:
                 click.echo("Downloaded {}".format(downloaded_file))
 
     else:
-        # register downloader
-        for idx, product in enumerate(search_results):
-            if product.downloader is None:
-                downloader = satim_api._plugins_manager.get_download_plugin(product)
-                auth = product.downloader_auth
-                if auth is None:
-                    auth = satim_api._plugins_manager.get_auth_plugin(
-                        downloader, product
-                    )
-                search_results[idx].register_downloader(downloader, auth)
-
-        downloaded_files = satim_api.download_all(search_results)
+        # Download products
+        downloaded_files = satim_api.download_all(search_results, output_dir=output_dir)
         if downloaded_files and len(downloaded_files) > 0:
             for downloaded_file in downloaded_files:
                 if downloaded_file is None:

eodag/config.py

@@ -46,6 +46,7 @@ from jsonpath_ng import JSONPath
 from eodag.api.product.metadata_mapping import mtd_cfg_as_conversion_and_querypath
 from eodag.utils import (
     HTTP_REQ_TIMEOUT,
+    STAC_SEARCH_PLUGINS,
     USER_AGENT,
     cached_yaml_load,
     cached_yaml_load_all,
@@ -819,12 +820,7 @@ def provider_config_init(
         if (
             stac_search_default_conf is not None
             and provider_config.search
-            and provider_config.search.type
-            in [
-                "StacSearch",
-                "StacListAssets",
-                "StaticStacSearch",
-            ]
+            and provider_config.search.type in STAC_SEARCH_PLUGINS
         ):
             # search config set to stac defaults overriden with provider config
             per_provider_stac_provider_config = deepcopy(stac_search_default_conf)

eodag/plugins/download/base.py

@@ -25,6 +25,7 @@ import tarfile
 import tempfile
 import zipfile
 from datetime import datetime, timedelta
+from pathlib import Path
 from time import sleep
 from typing import TYPE_CHECKING, Any, Callable, Optional, TypeVar, Union
 
@@ -232,12 +233,23 @@ class Download(PluginTopic):
         )
         if os.path.isfile(old_record_filename):
             os.rename(old_record_filename, record_filename)
-        if os.path.isfile(record_filename) and os.path.isfile(fs_path):
+
+        # path with or without extension
+        path_obj = Path(fs_path)
+        matched_paths = list(path_obj.parent.glob(f"{path_obj.stem}.*"))
+        fs_path_with_ext = matched_paths[0] if matched_paths else fs_path
+        if (
+            os.path.isfile(record_filename)
+            and fs_path_with_ext
+            and os.path.isfile(fs_path_with_ext)
+        ):
             logger.info(
-                f"Product already downloaded: {fs_path}",
+                f"Product already downloaded: {fs_path_with_ext}",
             )
             return (
-                self._finalize(fs_path, progress_callback=progress_callback, **kwargs),
+                self._finalize(
+                    str(fs_path_with_ext), progress_callback=progress_callback, **kwargs
+                ),
                 None,
             )
         elif os.path.isfile(record_filename) and os.path.isdir(fs_dir_path):

eodag/plugins/search/build_search_result.py

@@ -317,17 +317,17 @@ def append_time(input_date: date, time: Optional[str]) -> datetime:
 
 
 def parse_date(
-    date_str: str, time: Optional[Union[str, list[str]]]
+    date: str, time: Optional[Union[str, list[str]]]
 ) -> tuple[datetime, datetime]:
     """Parses a date string in formats YYYY-MM-DD, YYYMMDD, solo or in start/end or start/to/end intervals."""
-    if "to" in date_str:
-        start_date_str, end_date_str = date_str.split("/to/")
-    elif "/" in date_str:
-        dates = date_str.split("/")
+    if "to" in date:
+        start_date_str, end_date_str = date.split("/to/")
+    elif "/" in date:
+        dates = date.split("/")
         start_date_str = dates[0]
         end_date_str = dates[-1]
     else:
-        start_date_str = end_date_str = date_str
+        start_date_str = end_date_str = date
 
     # Update YYYYMMDD formatted dates
     if re.match(r"^\d{8}$", start_date_str):
@@ -401,6 +401,8 @@ def ecmwf_temporal_to_eodag(
     start = end = None
 
     if date := params.get("date"):
+        if isinstance(date, list):
+            date = "/".join(date)
         start, end = parse_date(date, params.get("time"))
 
     elif year := (params.get("year") or params.get("hyear")):

eodag/plugins/search/qssearch.py

@@ -1173,7 +1173,10 @@ class QueryStringSearch(Search):
 
         collection = getattr(self.config, "collection", None)
         if collection is None:
-            collection = prep.product_type_def_params.get("collection") or product_type
+            collection = (
+                getattr(prep, "product_type_def_params", {}).get("collection")
+                or product_type
+            )
 
         if collection is None:
             return ()