eodag 3.6.0__py3-none-any.whl → 3.8.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (35) hide show
  1. eodag/api/core.py +110 -189
  2. eodag/api/product/metadata_mapping.py +42 -3
  3. eodag/cli.py +6 -3
  4. eodag/config.py +7 -1
  5. eodag/plugins/authentication/openid_connect.py +1 -2
  6. eodag/plugins/download/aws.py +145 -178
  7. eodag/plugins/download/base.py +3 -2
  8. eodag/plugins/download/creodias_s3.py +10 -5
  9. eodag/plugins/download/http.py +14 -6
  10. eodag/plugins/download/s3rest.py +7 -3
  11. eodag/plugins/manager.py +1 -1
  12. eodag/plugins/search/base.py +34 -4
  13. eodag/plugins/search/build_search_result.py +3 -0
  14. eodag/plugins/search/cop_marine.py +2 -0
  15. eodag/plugins/search/data_request_search.py +6 -1
  16. eodag/plugins/search/qssearch.py +64 -25
  17. eodag/resources/ext_product_types.json +1 -1
  18. eodag/resources/product_types.yml +30 -171
  19. eodag/resources/providers.yml +87 -328
  20. eodag/resources/stac.yml +1 -2
  21. eodag/resources/stac_provider.yml +1 -1
  22. eodag/resources/user_conf_template.yml +0 -11
  23. eodag/rest/core.py +5 -16
  24. eodag/rest/stac.py +0 -4
  25. eodag/utils/__init__.py +41 -27
  26. eodag/utils/exceptions.py +4 -0
  27. eodag/utils/free_text_search.py +229 -0
  28. eodag/utils/s3.py +605 -65
  29. {eodag-3.6.0.dist-info → eodag-3.8.0.dist-info}/METADATA +7 -9
  30. {eodag-3.6.0.dist-info → eodag-3.8.0.dist-info}/RECORD +34 -34
  31. eodag/types/whoosh.py +0 -203
  32. {eodag-3.6.0.dist-info → eodag-3.8.0.dist-info}/WHEEL +0 -0
  33. {eodag-3.6.0.dist-info → eodag-3.8.0.dist-info}/entry_points.txt +0 -0
  34. {eodag-3.6.0.dist-info → eodag-3.8.0.dist-info}/licenses/LICENSE +0 -0
  35. {eodag-3.6.0.dist-info → eodag-3.8.0.dist-info}/top_level.txt +0 -0
@@ -25,6 +25,7 @@ from pydantic.fields import Field, FieldInfo
25
25
 
26
26
  from eodag.api.product.metadata_mapping import (
27
27
  DEFAULT_METADATA_MAPPING,
28
+ NOT_AVAILABLE,
28
29
  NOT_MAPPED,
29
30
  mtd_cfg_as_conversion_and_querypath,
30
31
  )
@@ -38,6 +39,7 @@ from eodag.utils import (
38
39
  copy_deepcopy,
39
40
  deepcopy,
40
41
  format_dict_items,
42
+ string_to_jsonpath,
41
43
  update_nested_dict,
42
44
  )
43
45
  from eodag.utils.exceptions import ValidationError
@@ -158,10 +160,6 @@ class Search(PluginTopic):
158
160
  :returns: The product type definition parameters
159
161
  """
160
162
  if product_type in self.config.products.keys():
161
- logger.debug(
162
- "Getting provider product type definition parameters for %s",
163
- product_type,
164
- )
165
163
  return self.config.products[product_type]
166
164
  elif GENERIC_PRODUCT_TYPE in self.config.products.keys():
167
165
  logger.debug(
@@ -439,3 +437,35 @@ class Search(PluginTopic):
439
437
  ):
440
438
  queryables[k] = v
441
439
  return queryables
440
+
441
+ def get_assets_from_mapping(self, provider_item: dict[str, Any]) -> dict[str, Any]:
442
+ """
443
+ Create assets based on the assets_mapping in the provider's config
444
+ and an item returned by the provider
445
+
446
+ :param provider_item: dict of item properties returned by the provider
447
+ :returns: dict containing the asset metadata
448
+ """
449
+ assets_mapping = getattr(self.config, "assets_mapping", None)
450
+ if not assets_mapping:
451
+ return {}
452
+ assets = {}
453
+ for key, values in assets_mapping.items():
454
+ asset_href = values.get("href")
455
+ if not asset_href:
456
+ logger.warning(
457
+ "asset mapping %s skipped because no href is available", key
458
+ )
459
+ continue
460
+ json_url_path = string_to_jsonpath(asset_href)
461
+ if isinstance(json_url_path, str):
462
+ url_path = json_url_path
463
+ else:
464
+ url_match = json_url_path.find(provider_item)
465
+ if len(url_match) == 1:
466
+ url_path = url_match[0].value
467
+ else:
468
+ url_path = NOT_AVAILABLE
469
+ assets[key] = deepcopy(values)
470
+ assets[key]["href"] = url_path
471
+ return assets
@@ -94,6 +94,7 @@ ECMWF_KEYWORDS = {
94
94
  "fcperiod",
95
95
  "fieldset",
96
96
  "filter",
97
+ "feature",
97
98
  "format",
98
99
  "frame",
99
100
  "frequency",
@@ -548,7 +549,9 @@ class ECMWFSearch(PostJsonSearch):
548
549
  :param params: Search parameters to be preprocessed.
549
550
  :param product_type: (optional) product type id
550
551
  """
552
+
551
553
  _dc_qs = params.get("_dc_qs")
554
+
552
555
  if _dc_qs is not None:
553
556
  # if available, update search params using datacube query-string
554
557
  _dc_qp = geojson.loads(unquote_plus(unquote_plus(_dc_qs)))
@@ -284,6 +284,8 @@ class CopMarineSearch(StaticStacSearch):
284
284
  "type": "application/x-netcdf",
285
285
  }
286
286
  }
287
+ additional_assets = self.get_assets_from_mapping(dataset_item)
288
+ assets.update(additional_assets)
287
289
  product = EOProduct(self.provider, properties, productType=product_type)
288
290
  # use product_type_config as default properties
289
291
  product_type_config = getattr(self.config, "product_type_config", {})
@@ -41,6 +41,7 @@ from eodag.utils import (
41
41
  GENERIC_PRODUCT_TYPE,
42
42
  HTTP_REQ_TIMEOUT,
43
43
  USER_AGENT,
44
+ _deprecated,
44
45
  deepcopy,
45
46
  string_to_jsonpath,
46
47
  )
@@ -57,6 +58,10 @@ if TYPE_CHECKING:
57
58
  logger = logging.getLogger("eodag.search.data_request_search")
58
59
 
59
60
 
61
+ @_deprecated(
62
+ reason="Plugin that was used in previous wekeo provider configuration, but not used anymore",
63
+ version="3.7.1",
64
+ )
60
65
  class DataRequestSearch(Search):
61
66
  """
62
67
  Plugin to execute search requests composed of several steps:
@@ -166,7 +171,7 @@ class DataRequestSearch(Search):
166
171
  data_request_id: Optional[str]
167
172
 
168
173
  def __init__(self, provider: str, config: PluginConfig) -> None:
169
- super(DataRequestSearch, self).__init__(provider, config)
174
+ super().__init__(provider, config)
170
175
  self.config.__dict__.setdefault("result_type", "json")
171
176
  self.config.__dict__.setdefault("results_entry", "content")
172
177
  self.config.__dict__.setdefault("pagination", {})
@@ -35,9 +35,11 @@ from typing import (
35
35
  from urllib.error import URLError
36
36
  from urllib.parse import (
37
37
  parse_qsl,
38
+ quote,
38
39
  quote_plus,
39
40
  unquote,
40
41
  unquote_plus,
42
+ urlencode,
41
43
  urlparse,
42
44
  urlunparse,
43
45
  )
@@ -86,10 +88,8 @@ from eodag.utils import (
86
88
  dict_items_recursive_apply,
87
89
  format_dict_items,
88
90
  get_ssl_context,
89
- quote,
90
91
  string_to_jsonpath,
91
92
  update_nested_dict,
92
- urlencode,
93
93
  )
94
94
  from eodag.utils.exceptions import (
95
95
  AuthenticationError,
@@ -386,36 +386,48 @@ class QueryStringSearch(Search):
386
386
 
387
387
  # parse jsonpath on init: product type specific metadata-mapping
388
388
  for product_type in self.config.products.keys():
389
- if "metadata_mapping" in self.config.products[product_type].keys():
390
- self.config.products[product_type][
391
- "metadata_mapping"
392
- ] = mtd_cfg_as_conversion_and_querypath(
393
- self.config.products[product_type]["metadata_mapping"]
394
- )
389
+
390
+ product_type_metadata_mapping = {}
391
+ # product-type specific metadata-mapping
392
+ if any(
393
+ mm in self.config.products[product_type].keys()
394
+ for mm in ("metadata_mapping", "metadata_mapping_from_product")
395
+ ):
395
396
  # Complete and ready to use product type specific metadata-mapping
396
397
  product_type_metadata_mapping = deepcopy(self.config.metadata_mapping)
397
398
 
398
- # update config using provider product type definition metadata_mapping
399
- # from another product
400
- other_product_for_mapping = cast(
401
- str,
402
- self.config.products[product_type].get(
403
- "metadata_mapping_from_product", ""
404
- ),
399
+ # metadata_mapping from another product
400
+ if other_product_for_mapping := self.config.products[product_type].get(
401
+ "metadata_mapping_from_product"
402
+ ):
403
+ other_product_type_def_params = self.get_product_type_def_params(
404
+ other_product_for_mapping,
405
405
  )
406
- if other_product_for_mapping:
407
- other_product_type_def_params = self.get_product_type_def_params(
408
- other_product_for_mapping,
409
- )
406
+ # parse mapping to apply
407
+ if other_product_type_mtd_mapping := other_product_type_def_params.get(
408
+ "metadata_mapping", {}
409
+ ):
410
410
  other_product_type_mtd_mapping = (
411
411
  mtd_cfg_as_conversion_and_querypath(
412
412
  other_product_type_def_params.get("metadata_mapping", {})
413
413
  )
414
414
  )
415
- # updated mapping at the end
416
- for metadata, mapping in other_product_type_mtd_mapping.items():
417
- product_type_metadata_mapping.pop(metadata, None)
418
- product_type_metadata_mapping[metadata] = mapping
415
+ else:
416
+ msg = f"Cannot reuse empty metadata_mapping from {other_product_for_mapping} for {product_type}"
417
+ raise MisconfiguredError(msg)
418
+ # update mapping
419
+ for metadata, mapping in other_product_type_mtd_mapping.items():
420
+ product_type_metadata_mapping.pop(metadata, None)
421
+ product_type_metadata_mapping[metadata] = mapping
422
+
423
+ # metadata_mapping from current product
424
+ if "metadata_mapping" in self.config.products[product_type].keys():
425
+ # parse mapping to apply
426
+ self.config.products[product_type][
427
+ "metadata_mapping"
428
+ ] = mtd_cfg_as_conversion_and_querypath(
429
+ self.config.products[product_type]["metadata_mapping"]
430
+ )
419
431
 
420
432
  # from current product, updated mapping at the end
421
433
  for metadata, mapping in self.config.products[product_type][
@@ -424,6 +436,7 @@ class QueryStringSearch(Search):
424
436
  product_type_metadata_mapping.pop(metadata, None)
425
437
  product_type_metadata_mapping[metadata] = mapping
426
438
 
439
+ if product_type_metadata_mapping:
427
440
  self.config.products[product_type][
428
441
  "metadata_mapping"
429
442
  ] = product_type_metadata_mapping
@@ -614,6 +627,26 @@ class QueryStringSearch(Search):
614
627
  generic_product_type_id
615
628
  ].update(collection_data)
616
629
 
630
+ # update product type id if needed
631
+ if collection_data_id := collection_data.get("ID"):
632
+ if generic_product_type_id != collection_data_id:
633
+ logger.debug(
634
+ "Rename %s product type to %s",
635
+ generic_product_type_id,
636
+ collection_data_id,
637
+ )
638
+ conf_update_dict["providers_config"][
639
+ collection_data_id
640
+ ] = conf_update_dict["providers_config"].pop(
641
+ generic_product_type_id
642
+ )
643
+ conf_update_dict["product_types_config"][
644
+ collection_data_id
645
+ ] = conf_update_dict["product_types_config"].pop(
646
+ generic_product_type_id
647
+ )
648
+ generic_product_type_id = collection_data_id
649
+
617
650
  # update keywords
618
651
  keywords_fields = [
619
652
  "instrument",
@@ -1090,6 +1123,8 @@ class QueryStringSearch(Search):
1090
1123
  product.properties = dict(
1091
1124
  getattr(self.config, "product_type_config", {}), **product.properties
1092
1125
  )
1126
+ additional_assets = self.get_assets_from_mapping(result)
1127
+ product.assets.update(additional_assets)
1093
1128
  # move assets from properties to product's attr, normalize keys & roles
1094
1129
  for key, asset in product.properties.pop("assets", {}).items():
1095
1130
  norm_key, asset["roles"] = product.driver.guess_asset_key_and_roles(
@@ -1481,13 +1516,17 @@ class PostJsonSearch(QueryStringSearch):
1481
1516
  )
1482
1517
 
1483
1518
  # Add to the query, the queryable parameters set in the provider product type definition
1519
+ product_type_metadata_mapping = {
1520
+ **getattr(self.config, "metadata_mapping", {}),
1521
+ **prep.product_type_def_params.get("metadata_mapping", {}),
1522
+ }
1484
1523
  keywords.update(
1485
1524
  {
1486
1525
  k: v
1487
1526
  for k, v in prep.product_type_def_params.items()
1488
1527
  if k not in keywords.keys()
1489
- and k in self.config.metadata_mapping.keys()
1490
- and isinstance(self.config.metadata_mapping[k], list)
1528
+ and k in product_type_metadata_mapping.keys()
1529
+ and isinstance(product_type_metadata_mapping[k], list)
1491
1530
  }
1492
1531
  )
1493
1532