eodag 3.6.0__py3-none-any.whl → 3.7.0__py3-none-any.whl

eodag/plugins/download/creodias_s3.py

@@ -15,7 +15,7 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-from typing import Optional
+from typing import TYPE_CHECKING, Optional
 
 import boto3
 from botocore.exceptions import ClientError
@@ -24,6 +24,9 @@ from eodag import EOProduct
 from eodag.plugins.download.aws import AwsDownload
 from eodag.utils.exceptions import MisconfiguredError
 
+if TYPE_CHECKING:
+    from mypy_boto3_s3.service_resource import S3ServiceResource
+
 
 class CreodiasS3Download(AwsDownload):
     """
@@ -60,19 +63,21 @@ class CreodiasS3Download(AwsDownload):
             )
 
         s3_session = boto3.session.Session(**auth_dict)
-        s3_resource = s3_session.resource(
+        s3_resource: S3ServiceResource = s3_session.resource(
             "s3", endpoint_url=getattr(self.config, "s3_endpoint", None)
         )
         objects = s3_resource.Bucket(bucket_name).objects.filter()
         list(objects.filter(Prefix=prefix).limit(1))
         self.s3_session = s3_session
+        self.s3_resource = s3_resource
         return objects
 
     def _get_bucket_names_and_prefixes(
         self,
         product: EOProduct,
-        asset_filter: Optional[str] = None,
-        ignore_assets: Optional[bool] = False,
+        asset_filter: Optional[str],
+        ignore_assets: bool,
+        complementary_url_keys: list[str],
     ) -> list[tuple[str, Optional[str]]]:
         """
         Retrieves the bucket names and path prefixes for the assets
@@ -85,7 +90,7 @@ class CreodiasS3Download(AwsDownload):
         # if assets are defined, use them instead of scanning product.location
         if len(product.assets) > 0 and not ignore_assets:
             bucket_names_and_prefixes = super()._get_bucket_names_and_prefixes(
-                product, asset_filter, ignore_assets
+                product, asset_filter, ignore_assets, complementary_url_keys
             )
         else:
             # if no assets are given, use productIdentifier to get S3 path for download

eodag/plugins/download/http.py

@@ -28,7 +28,16 @@ from email.message import Message
 from itertools import chain
 from json import JSONDecodeError
 from pathlib import Path
-from typing import TYPE_CHECKING, Any, Iterator, Optional, TypedDict, Union, cast
+from typing import (
+    TYPE_CHECKING,
+    Any,
+    Iterator,
+    Literal,
+    Optional,
+    TypedDict,
+    Union,
+    cast,
+)
 from urllib.parse import parse_qs, urlparse
 
 import geojson
@@ -745,7 +754,8 @@ class HTTPDownload(Download):
         self,
         product: EOProduct,
         auth: Optional[Union[AuthBase, S3SessionKwargs]] = None,
-        progress_callback: Optional[ProgressCallback] = None,
+        byte_range: tuple[Optional[int], Optional[int]] = (None, None),
+        compress: Literal["zip", "raw", "auto"] = "auto",
         wait: float = DEFAULT_DOWNLOAD_WAIT,
         timeout: float = DEFAULT_DOWNLOAD_TIMEOUT,
         **kwargs: Unpack[DownloadConf],
@@ -779,7 +789,7 @@ class HTTPDownload(Download):
                 chunks_tuples = self._stream_download_assets(
                     product,
                     auth,
-                    progress_callback,
+                    None,
                     assets_values=assets_values,
                     **kwargs,
                 )
@@ -825,9 +835,7 @@ class HTTPDownload(Download):
                 else:
                     pass
 
-        chunk_iterator = self._stream_download(
-            product, auth, progress_callback, **kwargs
-        )
+        chunk_iterator = self._stream_download(product, auth, None, **kwargs)
 
         # start reading chunks to set product.headers
         try:

eodag/plugins/download/s3rest.py

@@ -21,6 +21,7 @@ import logging
 import os
 import os.path
 from typing import TYPE_CHECKING, Optional, Union
+from urllib.parse import unquote, urljoin
 from xml.dom import minidom
 from xml.parsers.expat import ExpatError
 
@@ -40,8 +41,6 @@ from eodag.utils import (
     ProgressCallback,
     get_bucket_name_and_prefix,
     path_to_uri,
-    unquote,
-    urljoin,
 )
 from eodag.utils.exceptions import (
     AuthenticationError,

eodag/plugins/manager.py

@@ -109,7 +109,7 @@ class PluginManager:
                     logger.warning("Reason:\n%s", tb.format_exc())
                     logger.warning(
                         "Check that the plugin module (%s) is importable",
-                        entry_point.module_name,
+                        entry_point.name,
                     )
                 if entry_point.dist and entry_point.dist.name != "eodag":
                     # use plugin providers if any

eodag/plugins/search/base.py

@@ -25,6 +25,7 @@ from pydantic.fields import Field, FieldInfo
 
 from eodag.api.product.metadata_mapping import (
     DEFAULT_METADATA_MAPPING,
+    NOT_AVAILABLE,
     NOT_MAPPED,
     mtd_cfg_as_conversion_and_querypath,
 )
@@ -38,6 +39,7 @@ from eodag.utils import (
     copy_deepcopy,
     deepcopy,
     format_dict_items,
+    string_to_jsonpath,
     update_nested_dict,
 )
 from eodag.utils.exceptions import ValidationError
@@ -158,10 +160,6 @@ class Search(PluginTopic):
         :returns: The product type definition parameters
         """
         if product_type in self.config.products.keys():
-            logger.debug(
-                "Getting provider product type definition parameters for %s",
-                product_type,
-            )
             return self.config.products[product_type]
         elif GENERIC_PRODUCT_TYPE in self.config.products.keys():
             logger.debug(
@@ -439,3 +437,35 @@ class Search(PluginTopic):
             ):
                 queryables[k] = v
         return queryables
+
+    def get_assets_from_mapping(self, provider_item: dict[str, Any]) -> dict[str, Any]:
+        """
+        Create assets based on the assets_mapping in the provider's config
+        and an item returned by the provider
+
+        :param provider_item: dict of item properties returned by the provider
+        :returns: dict containing the asset metadata
+        """
+        assets_mapping = getattr(self.config, "assets_mapping", None)
+        if not assets_mapping:
+            return {}
+        assets = {}
+        for key, values in assets_mapping.items():
+            asset_href = values.get("href")
+            if not asset_href:
+                logger.warning(
+                    "asset mapping %s skipped because no href is available", key
+                )
+                continue
+            json_url_path = string_to_jsonpath(asset_href)
+            if isinstance(json_url_path, str):
+                url_path = json_url_path
+            else:
+                url_match = json_url_path.find(provider_item)
+                if len(url_match) == 1:
+                    url_path = url_match[0].value
+                else:
+                    url_path = NOT_AVAILABLE
+            assets[key] = deepcopy(values)
+            assets[key]["href"] = url_path
+        return assets

eodag/plugins/search/build_search_result.py

@@ -94,6 +94,7 @@ ECMWF_KEYWORDS = {
     "fcperiod",
     "fieldset",
     "filter",
+    "feature",
     "format",
     "frame",
     "frequency",
@@ -548,7 +549,9 @@ class ECMWFSearch(PostJsonSearch):
         :param params: Search parameters to be preprocessed.
         :param product_type: (optional) product type id
         """
+
         _dc_qs = params.get("_dc_qs")
+
         if _dc_qs is not None:
             # if available, update search params using datacube query-string
             _dc_qp = geojson.loads(unquote_plus(unquote_plus(_dc_qs)))

eodag/plugins/search/cop_marine.py

@@ -284,6 +284,8 @@ class CopMarineSearch(StaticStacSearch):
                 "type": "application/x-netcdf",
             }
         }
+        additional_assets = self.get_assets_from_mapping(dataset_item)
+        assets.update(additional_assets)
         product = EOProduct(self.provider, properties, productType=product_type)
         # use product_type_config as default properties
         product_type_config = getattr(self.config, "product_type_config", {})

eodag/plugins/search/qssearch.py

@@ -35,9 +35,11 @@ from typing import (
 from urllib.error import URLError
 from urllib.parse import (
     parse_qsl,
+    quote,
     quote_plus,
     unquote,
     unquote_plus,
+    urlencode,
     urlparse,
     urlunparse,
 )
@@ -86,10 +88,8 @@ from eodag.utils import (
     dict_items_recursive_apply,
     format_dict_items,
     get_ssl_context,
-    quote,
     string_to_jsonpath,
     update_nested_dict,
-    urlencode,
 )
 from eodag.utils.exceptions import (
     AuthenticationError,
@@ -386,36 +386,48 @@ class QueryStringSearch(Search):
 
         # parse jsonpath on init: product type specific metadata-mapping
         for product_type in self.config.products.keys():
-            if "metadata_mapping" in self.config.products[product_type].keys():
-                self.config.products[product_type][
-                    "metadata_mapping"
-                ] = mtd_cfg_as_conversion_and_querypath(
-                    self.config.products[product_type]["metadata_mapping"]
-                )
+
+            product_type_metadata_mapping = {}
+            # product-type specific metadata-mapping
+            if any(
+                mm in self.config.products[product_type].keys()
+                for mm in ("metadata_mapping", "metadata_mapping_from_product")
+            ):
                 # Complete and ready to use product type specific metadata-mapping
                 product_type_metadata_mapping = deepcopy(self.config.metadata_mapping)
 
-                # update config using provider product type definition metadata_mapping
-                # from another product
-                other_product_for_mapping = cast(
-                    str,
-                    self.config.products[product_type].get(
-                        "metadata_mapping_from_product", ""
-                    ),
+            # metadata_mapping from another product
+            if other_product_for_mapping := self.config.products[product_type].get(
+                "metadata_mapping_from_product"
+            ):
+                other_product_type_def_params = self.get_product_type_def_params(
+                    other_product_for_mapping,
                 )
-                if other_product_for_mapping:
-                    other_product_type_def_params = self.get_product_type_def_params(
-                        other_product_for_mapping,
-                    )
+                # parse mapping to apply
+                if other_product_type_mtd_mapping := other_product_type_def_params.get(
+                    "metadata_mapping", {}
+                ):
                     other_product_type_mtd_mapping = (
                         mtd_cfg_as_conversion_and_querypath(
                             other_product_type_def_params.get("metadata_mapping", {})
                         )
                     )
-                    # updated mapping at the end
-                    for metadata, mapping in other_product_type_mtd_mapping.items():
-                        product_type_metadata_mapping.pop(metadata, None)
-                        product_type_metadata_mapping[metadata] = mapping
+                else:
+                    msg = f"Cannot reuse empty metadata_mapping from {other_product_for_mapping} for {product_type}"
+                    raise MisconfiguredError(msg)
+                # update mapping
+                for metadata, mapping in other_product_type_mtd_mapping.items():
+                    product_type_metadata_mapping.pop(metadata, None)
+                    product_type_metadata_mapping[metadata] = mapping
+
+            # metadata_mapping from current product
+            if "metadata_mapping" in self.config.products[product_type].keys():
+                # parse mapping to apply
+                self.config.products[product_type][
+                    "metadata_mapping"
+                ] = mtd_cfg_as_conversion_and_querypath(
+                    self.config.products[product_type]["metadata_mapping"]
+                )
 
                 # from current product, updated mapping at the end
                 for metadata, mapping in self.config.products[product_type][
@@ -424,6 +436,7 @@ class QueryStringSearch(Search):
                     product_type_metadata_mapping.pop(metadata, None)
                     product_type_metadata_mapping[metadata] = mapping
 
+            if product_type_metadata_mapping:
                 self.config.products[product_type][
                     "metadata_mapping"
                 ] = product_type_metadata_mapping
@@ -1090,6 +1103,8 @@ class QueryStringSearch(Search):
             product.properties = dict(
                 getattr(self.config, "product_type_config", {}), **product.properties
             )
+            additional_assets = self.get_assets_from_mapping(result)
+            product.assets.update(additional_assets)
             # move assets from properties to product's attr, normalize keys & roles
             for key, asset in product.properties.pop("assets", {}).items():
                 norm_key, asset["roles"] = product.driver.guess_asset_key_and_roles(
@@ -1481,13 +1496,17 @@ class PostJsonSearch(QueryStringSearch):
             )
 
             # Add to the query, the queryable parameters set in the provider product type definition
+            product_type_metadata_mapping = {
+                **getattr(self.config, "metadata_mapping", {}),
+                **prep.product_type_def_params.get("metadata_mapping", {}),
+            }
             keywords.update(
                 {
                     k: v
                     for k, v in prep.product_type_def_params.items()
                     if k not in keywords.keys()
-                    and k in self.config.metadata_mapping.keys()
-                    and isinstance(self.config.metadata_mapping[k], list)
+                    and k in product_type_metadata_mapping.keys()
+                    and isinstance(product_type_metadata_mapping[k], list)
                 }
             )