eodag 4.0.0a4__py3-none-any.whl → 4.0.0b1__py3-none-any.whl

eodag/api/collection.py

@@ -20,7 +20,7 @@ from __future__ import annotations
 import logging
 import re
 from collections import UserDict, UserList
-from typing import TYPE_CHECKING, Any, Optional
+from typing import TYPE_CHECKING, Any, Optional, cast
 
 from pydantic import BaseModel, ConfigDict, Field, PrivateAttr
 from pydantic import ValidationError as PydanticValidationError
@@ -29,6 +29,9 @@ from pydantic_core import ErrorDetails, InitErrorDetails, PydanticCustomError
 from stac_pydantic.collection import Extent, Provider, SpatialExtent, TimeInterval
 from stac_pydantic.links import Links
 
+from eodag.types.queryables import CommonStacMetadata
+from eodag.types.stac_metadata import create_stac_metadata_model
+from eodag.utils import STAC_VERSION
 from eodag.utils.env import is_env_var_true
 from eodag.utils.exceptions import ValidationError
 from eodag.utils.repr import dict_to_html_table
@@ -91,6 +94,11 @@ class Collection(BaseModel):
         repr=False,
     )
 
+    # path to external collection metadata file (required by stac-fastapi-eodag)
+    eodag_stac_collection: Optional[str] = Field(
+        default=None, alias="stacCollection", exclude=True, repr=False
+    )
+
     # Private property to store the eodag internal id value. Not part of the model schema.
     _id: str = PrivateAttr()
     _dag: Optional[EODataAccessGateway] = PrivateAttr(default=None)
@@ -276,6 +284,62 @@ class Collection(BaseModel):
 
         return dag.list_queryables(collection=self.id, **kwargs)
 
+    def serialize(self) -> dict[str, Any]:
+        """Serialize the Collection instance to a STAC dictionary.
+
+        :returns: A STAC dictionary representation of the Collection instance.
+        """
+        stac_dict: dict[str, Any] = {
+            "stac_version": STAC_VERSION,
+            "type": "Collection",
+        }
+
+        stac_dict |= self.model_dump(mode="json", exclude_none=True, exclude={"alias"})
+
+        stac_dict.setdefault("links", [])
+        stac_dict.setdefault("providers", [])
+
+        not_in_summaries = [
+            "stac_version",
+            "type",
+            "id",
+            "title",
+            "description",
+            "extent",
+            "keywords",
+            "license",
+            "links",
+            "providers",
+        ]
+        summaries = dict()
+        for k, v in stac_dict.items():
+            if k not in not_in_summaries:
+                if isinstance(v, list):
+                    summaries[k] = v
+                elif isinstance(v, str):
+                    summaries[k] = v.split(",")
+                else:
+                    summaries[k] = [v]
+        stac_dict["summaries"] = summaries
+
+        # Remove empty items and items moved to summaries
+        keys_to_remove = [
+            k
+            for k in stac_dict.keys()
+            if k not in not_in_summaries and k != "summaries"
+        ]
+        for k in keys_to_remove:
+            del stac_dict[k]
+
+        # add extensions
+        summaries_model = cast(CommonStacMetadata, create_stac_metadata_model())
+        summaries_validated = summaries_model.model_construct(
+            _fields_set=None, **summaries
+        )
+        stac_dict["stac_extensions"] = summaries_validated.get_conformance_classes()
+
+        return stac_dict
+
 
 class CollectionsDict(UserDict[str, Collection]):
     """A UserDict object which values are :class:`~eodag.api.collection.Collection` objects, keyed by provider ``id``.

eodag/api/core.py

@@ -30,6 +30,7 @@ from copy import deepcopy
 from importlib.metadata import version
 from importlib.resources import files as res_files
 from operator import attrgetter, itemgetter
+from pathlib import Path
 from typing import TYPE_CHECKING, Any, Iterator, Optional, Union, cast
 
 import geojson
@@ -87,6 +88,7 @@ from eodag.utils.free_text_search import compile_free_text_query
 from eodag.utils.stac_reader import fetch_stac_items
 
 if TYPE_CHECKING:
+    from concurrent.futures import ThreadPoolExecutor
     from shapely.geometry.base import BaseGeometry
 
     from eodag.api.product import EOProduct
@@ -153,11 +155,14 @@ class EODataAccessGateway:
             user_conf_file_path = os.getenv(env_var_name)
             if user_conf_file_path is None:
                 user_conf_file_path = standard_configuration_path
-                if not os.path.isfile(standard_configuration_path):
+                source = str(
+                    res_files("eodag") / "resources" / "user_conf_template.yml"
+                )
+                if os.path.isfile(source) and not os.path.isfile(
+                    standard_configuration_path
+                ):
                     shutil.copy(
-                        str(
-                            res_files("eodag") / "resources" / "user_conf_template.yml"
-                        ),
+                        source,
                         standard_configuration_path,
                     )
         self._providers.update_from_config_file(user_conf_file_path)
@@ -452,6 +457,9 @@ class EODataAccessGateway:
             if locations_conf_path is None:
                 locations_conf_path = os.path.join(self.conf_dir, "locations.yml")
                 if not os.path.isfile(locations_conf_path):
+                    # Ensure the directory exists
+                    os.makedirs(os.path.dirname(locations_conf_path), exist_ok=True)
+
                     # copy locations conf file and replace path example
                     locations_conf_template = str(
                         res_files("eodag") / "resources" / "locations_conf_template.yml"
@@ -1779,14 +1787,15 @@ class EODataAccessGateway:
 
             # remove None values and convert param names to their pydantic alias if any
             search_params = {}
+            queryables_fields = Queryables.from_stac_models().model_fields
             ecmwf_queryables = [
                 f"{ECMWF_PREFIX[:-1]}_{k}" for k in ECMWF_ALLOWED_KEYWORDS
             ]
             for param, value in kwargs.items():
                 if value is None:
                     continue
-                if param in Queryables.model_fields:
-                    param_alias = Queryables.model_fields[param].alias or param
+                if param in queryables_fields:
+                    param_alias = queryables_fields[param].alias or param
                     search_params[param_alias] = value
                 elif param in ecmwf_queryables:
                     # alias equivalent for ECMWF queryables
@@ -1845,14 +1854,6 @@ class EODataAccessGateway:
                     else:
                         eo_product.collection = guesses[0].id
 
-                try:
-                    if eo_product.collection is not None:
-                        eo_product.collection = self.get_collection_from_alias(
-                            eo_product.collection
-                        )
-                except NoMatchingCollection:
-                    logger.debug("collection %s not found", eo_product.collection)
-
                 if eo_product.search_intersection is not None:
                     eo_product._register_downloader_from_manager(self._plugins_manager)
 
@@ -1919,6 +1920,7 @@ class EODataAccessGateway:
         search_result: SearchResult,
         downloaded_callback: Optional[DownloadedCallback] = None,
         progress_callback: Optional[ProgressCallback] = None,
+        executor: Optional[ThreadPoolExecutor] = None,
         wait: float = DEFAULT_DOWNLOAD_WAIT,
         timeout: float = DEFAULT_DOWNLOAD_TIMEOUT,
         **kwargs: Unpack[DownloadConf],
@@ -1936,6 +1938,8 @@ class EODataAccessGateway:
                                   size as inputs and handle progress bar
                                   creation and update to give the user a
                                   feedback on the download progress
+        :param executor: (optional) An executor to download EO products of ``search_result`` in parallel
+                                    which will also be reused to download assets of these products in parallel.
         :param wait: (optional) If download fails, wait time in minutes between
                      two download tries of the same product
         :param timeout: (optional) If download fails, maximum time in minutes
@@ -1956,8 +1960,7 @@ class EODataAccessGateway:
         paths = []
         if search_result:
             logger.info("Downloading %s products", len(search_result))
-            # Get download plugin using first product assuming product from several provider
-            # aren't mixed into a search result
+            # Get download plugin using first product assuming all plugins use base.Download.download_all
             download_plugin = self._plugins_manager.get_download_plugin(
                 search_result[0]
             )
@@ -1965,6 +1968,7 @@ class EODataAccessGateway:
                 search_result,
                 downloaded_callback=downloaded_callback,
                 progress_callback=progress_callback,
+                executor=executor,
                 wait=wait,
                 timeout=timeout,
                 **kwargs,
@@ -1986,8 +1990,42 @@ class EODataAccessGateway:
         :param filename: (optional) The name of the file to generate
         :returns: The name of the created file
         """
+        search_result_dict = search_result.as_geojson_object()
+        # add self link
+        search_result_dict.setdefault("links", [])
+        search_result_dict["links"].append(
+            {
+                "rel": "self",
+                "href": f"{filename}",
+                "type": "application/json",
+            },
+        )
+        # write search results
         with open(filename, "w") as fh:
-            geojson.dump(search_result.as_geojson_object(), fh)
+            geojson.dump(search_result_dict, fh)
+            logger.debug("Search results saved to %s", filename)
+        # write collection(s)
+        if search_result._dag is None:
+            return filename
+        collections = set(p.collection for p in search_result)
+        for collection in collections:
+            collection_obj = search_result._dag.collections_config.get(
+                collection, Collection(id=collection)
+            )
+            collection_dict = collection_obj.serialize()
+            # add links
+            collection_dict.setdefault("links", [])
+            collection_dict["links"].append(
+                {
+                    "rel": "self",
+                    "href": f"{collection}.json",
+                    "type": "application/json",
+                },
+            )
+            with open(Path(filename).parent / f"{collection}.json", "w") as fh:
+                geojson.dump(collection_dict, fh)
+                logger.debug("Collection '%s' saved to %s", collection, fh.name)
+
         return filename
 
     @staticmethod
@@ -2026,6 +2064,7 @@ class EODataAccessGateway:
         self,
         product: EOProduct,
         progress_callback: Optional[ProgressCallback] = None,
+        executor: Optional[ThreadPoolExecutor] = None,
         wait: float = DEFAULT_DOWNLOAD_WAIT,
         timeout: float = DEFAULT_DOWNLOAD_TIMEOUT,
         **kwargs: Unpack[DownloadConf],
@@ -2056,6 +2095,8 @@ class EODataAccessGateway:
                                   size as inputs and handle progress bar
                                   creation and update to give the user a
                                   feedback on the download progress
+        :param executor: (optional) An executor to download assets of ``product`` in parallel if it has any. If ``None``
+                         , a default executor will be created
         :param wait: (optional) If download fails, wait time in minutes between
                     two download tries
         :param timeout: (optional) If download fails, maximum time in minutes
@@ -2080,7 +2121,11 @@ class EODataAccessGateway:
             return uri_to_path(product.location)
         self._setup_downloader(product)
         path = product.download(
-            progress_callback=progress_callback, wait=wait, timeout=timeout, **kwargs
+            progress_callback=progress_callback,
+            executor=executor,
+            wait=wait,
+            timeout=timeout,
+            **kwargs,
         )
 
         return path
@@ -2184,7 +2229,8 @@ class EODataAccessGateway:
 
             # use queryables aliases
             kwargs_alias = {**kwargs}
-            for search_param, field_info in Queryables.model_fields.items():
+            queryables_fields = Queryables.from_stac_models().model_fields
+            for search_param, field_info in queryables_fields.items():
                 if search_param in kwargs and field_info.alias:
                     kwargs_alias[field_info.alias] = kwargs_alias.pop(search_param)
 

eodag/api/product/_assets.py

@@ -129,7 +129,7 @@ class AssetsDict(UserDict):
                         ...
                     }}
                 </summary>
-                    {dict_to_html_table(v, depth=1)}
+                    {dict_to_html_table(v, depth=2)}
                 </details>
                 </td></tr>
                 """

eodag/api/product/_product.py

@@ -23,14 +23,18 @@ import os
 import re
 import tempfile
 from datetime import datetime
-from typing import TYPE_CHECKING, Any, Optional, Union
+from typing import TYPE_CHECKING, Any, Iterable, Optional, Union, cast
 
+import orjson
 import requests
 from requests import RequestException
 from requests.auth import AuthBase
 from shapely import geometry
 from shapely.errors import ShapelyError
 
+from eodag.types.queryables import CommonStacMetadata
+from eodag.types.stac_metadata import create_stac_metadata_model
+
 try:
     # import from eodag-cube if installed
     from eodag_cube.api.product import (  # pyright: ignore[reportMissingImports]
@@ -52,6 +56,7 @@ from eodag.utils import (
     DEFAULT_DOWNLOAD_WAIT,
     DEFAULT_SHAPELY_GEOMETRY,
     DEFAULT_STREAM_REQUESTS_TIMEOUT,
+    STAC_VERSION,
     USER_AGENT,
     ProgressCallback,
     format_string,
@@ -61,6 +66,7 @@ from eodag.utils.exceptions import DownloadError, MisconfiguredError, Validation
 from eodag.utils.repr import dict_to_html_table
 
 if TYPE_CHECKING:
+    from concurrent.futures import ThreadPoolExecutor
     from shapely.geometry.base import BaseGeometry
 
     from eodag.api.product.drivers.base import DatasetDriver
@@ -122,6 +128,8 @@ class EOProduct:
     search_kwargs: Any
     #: Datetime for download next try
     next_try: datetime
+    #: Stream for requests
+    _stream: requests.Response
 
     def __init__(
         self, provider: str, properties: dict[str, Any], **kwargs: Any
@@ -143,6 +151,13 @@ class EOProduct:
             and not key.startswith("_")
             and value is not None
         }
+        self.properties.setdefault(
+            "datetime",
+            self.properties.get("start_datetime")
+            or self.properties.get("end_datetime"),
+        )
+
+        # sort properties to have common stac properties first
         common_stac_properties = {
             key: self.properties[key]
             for key in sorted(self.properties)
@@ -202,25 +217,71 @@ class EOProduct:
         """
         search_intersection = None
         if self.search_intersection is not None:
-            search_intersection = geometry.mapping(self.search_intersection)
+            search_intersection = orjson.loads(
+                orjson.dumps(self.search_intersection.__geo_interface__)
+            )
+
+        # product properties
+        stac_properties = {
+            **{
+                key: value
+                for key, value in self.properties.items()
+                if key not in ("geometry", "id")
+            },
+            "eodag:provider": self.provider,
+            "eodag:search_intersection": search_intersection,
+        }
+        stac_providers = self.properties.get("providers", [])
+        if not any("host" in p.get("roles", []) for p in stac_providers):
+            stac_providers.append({"name": self.provider, "roles": ["host"]})
+            stac_properties["providers"] = stac_providers
+
+        props_model = cast(type[CommonStacMetadata], create_stac_metadata_model())
+        props_validated = props_model.safe_validate(stac_properties)
+        stac_extensions: set[str] = set(props_validated.get_conformance_classes())
+
+        # skip invalid properties
+        invalid_properties = {
+            k
+            for k in stac_properties.keys()
+            if k not in props_validated.to_dict() and props_model.has_field(k)
+        }
+        for key in invalid_properties:
+            stac_properties.pop(key, None)
+
+        # get conformance classes for assets properties
+        assets_dict = {**self.assets.as_dict()}
+        for asset_key, asset_properties in self.assets.as_dict().items():
+            asset_props_validated = props_model.safe_validate(asset_properties)
+            stac_extensions.update(asset_props_validated.get_conformance_classes())
+
+            # skip invalid assets properties
+            invalid_asset_properties = {
+                k
+                for k in asset_properties.keys()
+                if k not in asset_props_validated.to_dict() and props_model.has_field(k)
+            }
+            for key in invalid_asset_properties:
+                assets_dict[asset_key].pop(key, None)
 
         geojson_repr: dict[str, Any] = {
             "type": "Feature",
-            "geometry": geometry.mapping(self.geometry),
+            "geometry": orjson.loads(orjson.dumps(self.geometry.__geo_interface__)),
+            "bbox": list(self.geometry.bounds),
             "id": self.properties["id"],
-            "assets": self.assets.as_dict(),
-            "properties": {
-                "eodag:collection": self.collection,
-                "eodag:provider": self.provider,
-                "eodag:search_intersection": search_intersection,
-                **{
-                    key: value
-                    for key, value in self.properties.items()
-                    if key not in ("geometry", "id")
+            "assets": assets_dict,
+            "properties": stac_properties,
+            "links": [
+                {
+                    "rel": "collection",
+                    "href": f"{self.collection}.json",
+                    "type": "application/json",
                 },
-            },
+            ],
+            "stac_extensions": list(stac_extensions),
+            "stac_version": STAC_VERSION,
+            "collection": self.collection,
         }
-
         return geojson_repr
 
     @classmethod
@@ -234,11 +295,11 @@ class EOProduct:
         :raises: :class:`~eodag.utils.exceptions.ValidationError`
         """
         try:
+            collection = feature.get("collection")
             properties = feature["properties"]
             properties["geometry"] = feature["geometry"]
             properties["id"] = feature["id"]
             provider = properties.pop("eodag:provider")
-            collection = properties.pop("eodag:collection")
             search_intersection = properties.pop("eodag:search_intersection")
         except KeyError as e:
             raise ValidationError(
@@ -337,6 +398,7 @@ class EOProduct:
     def download(
         self,
         progress_callback: Optional[ProgressCallback] = None,
+        executor: Optional[ThreadPoolExecutor] = None,
         wait: float = DEFAULT_DOWNLOAD_WAIT,
         timeout: float = DEFAULT_DOWNLOAD_TIMEOUT,
         **kwargs: Unpack[DownloadConf],
@@ -353,6 +415,8 @@ class EOProduct:
                                   size as inputs and handle progress bar
                                   creation and update to give the user a
                                   feedback on the download progress
+        :param executor: (optional) An executor to download assets of the product in parallel if it has any. If ``None``
+                         , a default executor will be created
         :param wait: (optional) If download fails, wait time in minutes between
                      two download tries
         :param timeout: (optional) If download fails, maximum time in minutes
@@ -377,17 +441,26 @@ class EOProduct:
         )
 
         progress_callback, close_progress_callback = self._init_progress_bar(
-            progress_callback
+            progress_callback, executor
         )
+
         fs_path = self.downloader.download(
             self,
             auth=auth,
             progress_callback=progress_callback,
+            executor=executor,
             wait=wait,
             timeout=timeout,
             **kwargs,
         )
 
+        # shutdown executor if it was not created during parallel product downloads
+        if (
+            executor is not None
+            and executor._thread_name_prefix != "eodag-download-all"
+        ):
+            executor.shutdown(wait=True)
+
         # close progress bar if needed
         if close_progress_callback:
             progress_callback.close()
@@ -408,15 +481,22 @@ class EOProduct:
         return fs_path
 
     def _init_progress_bar(
-        self, progress_callback: Optional[ProgressCallback]
+        self,
+        progress_callback: Optional[ProgressCallback],
+        executor: Optional[ThreadPoolExecutor],
     ) -> tuple[ProgressCallback, bool]:
+        # determine position of the progress bar with a counter of executor passings
+        # to avoid bar overwriting in case of parallel downloads
+        count = executor._counter() if executor is not None else 1  # type: ignore
+
         # progress bar init
         if progress_callback is None:
-            progress_callback = ProgressCallback(position=1)
+            progress_callback = ProgressCallback(position=count)
             # one shot progress callback to close after download
             close_progress_callback = True
         else:
             close_progress_callback = False
+            progress_callback.pos = count
             # update units as bar may have been previously used for extraction
             progress_callback.unit = "B"
             progress_callback.unit_scale = True
@@ -641,3 +721,38 @@ class EOProduct:
                     <td {thumbnail_style} title='properties[&quot;thumbnail&quot;]'>{thumbnail_html}</td>
                 </tr>
             </table>"""
+
+    def to_xarray(
+        self,
+        asset_key: Optional[str] = None,
+        wait: float = DEFAULT_DOWNLOAD_WAIT,
+        timeout: float = DEFAULT_DOWNLOAD_TIMEOUT,
+        roles: Iterable[str] = {"data", "data-mask"},
+        **xarray_kwargs: Any,
+    ):
+        """
+        Return product data as a dictionary of :class:`xarray.Dataset`.
+
+        :param asset_key: (optional) key of the asset. If not specified the whole
+                          product data will be retrieved
+        :param wait: (optional) If order is needed, wait time in minutes between two
+                     order status check
+        :param timeout: (optional) If order is needed, maximum time in minutes before
+                        stop checking order status
+        :param roles: (optional) roles of assets that must be fetched
+        :param xarray_kwargs: (optional) keyword arguments passed to :func:`xarray.open_dataset`
+        :returns: a dictionary of :class:`xarray.Dataset`
+        """
+        raise NotImplementedError("Install eodag-cube to make this method available.")
+
+    def augment_from_xarray(
+        self,
+        roles: Iterable[str] = {"data", "data-mask"},
+    ) -> EOProduct:
+        """
+        Annotate the product properties and assets with STAC metadata got by fetching its xarray representation.
+
+        :param roles: (optional) roles of assets that must be fetched
+        :returns: updated EOProduct
+        """
+        raise NotImplementedError("Install eodag-cube to make this method available.")

eodag/api/product/drivers/__init__.py

@@ -18,7 +18,9 @@
 """EODAG drivers package"""
 from __future__ import annotations
 
-from typing import Callable, TypedDict
+from typing import Callable
+
+from typing_extensions import TypedDict
 
 from eodag.api.product.drivers.base import DatasetDriver
 from eodag.api.product.drivers.generic import GenericDriver

eodag/api/product/drivers/base.py

@@ -19,7 +19,9 @@ from __future__ import annotations
 
 import logging
 import re
-from typing import TYPE_CHECKING, Optional, TypedDict
+from typing import TYPE_CHECKING, Optional
+
+from typing_extensions import TypedDict
 
 if TYPE_CHECKING:
     from eodag.api.product import EOProduct

eodag/api/product/drivers/generic.py

@@ -36,7 +36,7 @@ class GenericDriver(DatasetDriver):
                 (
                     r"^(?:.*[/\\])?([^/\\]+)"
                     r"(\.jp2|\.tiff?|\.dat|\.nc|\.grib2?|"
-                    r"\.zarr|\.nat|\.covjson|\.parquet|\.zip|\.tar|\.gz)$"
+                    r"\.zarr|\.nat|\.covjson|\.parquet|\.zip|\.tar|\.gz)(?:\?.*)?$"
                 ),
                 re.IGNORECASE,
             ),
@@ -45,25 +45,29 @@ class GenericDriver(DatasetDriver):
         # metadata
         {
             "pattern": re.compile(
-                r"^(?:.*[/\\])?([^/\\]+)(\.xml|\.xsd|\.safe|\.json)$", re.IGNORECASE
+                r"^(?:.*[/\\])?([^/\\]+)(\.xml|\.xsd|\.safe|\.json)(?:\?.*)?$",
+                re.IGNORECASE,
             ),
             "roles": ["metadata"],
         },
         # thumbnail
         {
             "pattern": re.compile(
-                r"^(?:.*[/\\])?(thumbnail)(\.jpg|\.jpeg|\.png)$", re.IGNORECASE
+                r"^(?:.*[/\\])?(thumbnail)(\.jpg|\.jpeg|\.png)(?:\?.*)?$", re.IGNORECASE
             ),
             "roles": ["thumbnail"],
         },
         # quicklook
         {
             "pattern": re.compile(
-                r"^(?:.*[/\\])?([^/\\]+-ql|preview|quick-?look)(\.jpg|\.jpeg|\.png)$",
+                r"^(?:.*[/\\])?([^/\\]+-ql|preview|quick-?look)(\.jpg|\.jpeg|\.png)(?:\?.*)?$",
                 re.IGNORECASE,
             ),
             "roles": ["overview"],
         },
         # default
-        {"pattern": re.compile(r"^(?:.*[/\\])?([^/\\]+)$"), "roles": ["auxiliary"]},
+        {
+            "pattern": re.compile(r"^(?:.*[/\\])?([^/\\?]+)(?:\?.*)?$"),
+            "roles": ["auxiliary"],
+        },
     ]