eodag 3.6.0__py3-none-any.whl → 3.8.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- eodag/api/core.py +110 -189
- eodag/api/product/metadata_mapping.py +42 -3
- eodag/cli.py +6 -3
- eodag/config.py +7 -1
- eodag/plugins/authentication/openid_connect.py +1 -2
- eodag/plugins/download/aws.py +145 -178
- eodag/plugins/download/base.py +3 -2
- eodag/plugins/download/creodias_s3.py +10 -5
- eodag/plugins/download/http.py +14 -6
- eodag/plugins/download/s3rest.py +7 -3
- eodag/plugins/manager.py +1 -1
- eodag/plugins/search/base.py +34 -4
- eodag/plugins/search/build_search_result.py +3 -0
- eodag/plugins/search/cop_marine.py +2 -0
- eodag/plugins/search/data_request_search.py +6 -1
- eodag/plugins/search/qssearch.py +64 -25
- eodag/resources/ext_product_types.json +1 -1
- eodag/resources/product_types.yml +30 -171
- eodag/resources/providers.yml +87 -328
- eodag/resources/stac.yml +1 -2
- eodag/resources/stac_provider.yml +1 -1
- eodag/resources/user_conf_template.yml +0 -11
- eodag/rest/core.py +5 -16
- eodag/rest/stac.py +0 -4
- eodag/utils/__init__.py +41 -27
- eodag/utils/exceptions.py +4 -0
- eodag/utils/free_text_search.py +229 -0
- eodag/utils/s3.py +605 -65
- {eodag-3.6.0.dist-info → eodag-3.8.0.dist-info}/METADATA +7 -9
- {eodag-3.6.0.dist-info → eodag-3.8.0.dist-info}/RECORD +34 -34
- eodag/types/whoosh.py +0 -203
- {eodag-3.6.0.dist-info → eodag-3.8.0.dist-info}/WHEEL +0 -0
- {eodag-3.6.0.dist-info → eodag-3.8.0.dist-info}/entry_points.txt +0 -0
- {eodag-3.6.0.dist-info → eodag-3.8.0.dist-info}/licenses/LICENSE +0 -0
- {eodag-3.6.0.dist-info → eodag-3.8.0.dist-info}/top_level.txt +0 -0
eodag/plugins/search/base.py
CHANGED
|
@@ -25,6 +25,7 @@ from pydantic.fields import Field, FieldInfo
|
|
|
25
25
|
|
|
26
26
|
from eodag.api.product.metadata_mapping import (
|
|
27
27
|
DEFAULT_METADATA_MAPPING,
|
|
28
|
+
NOT_AVAILABLE,
|
|
28
29
|
NOT_MAPPED,
|
|
29
30
|
mtd_cfg_as_conversion_and_querypath,
|
|
30
31
|
)
|
|
@@ -38,6 +39,7 @@ from eodag.utils import (
|
|
|
38
39
|
copy_deepcopy,
|
|
39
40
|
deepcopy,
|
|
40
41
|
format_dict_items,
|
|
42
|
+
string_to_jsonpath,
|
|
41
43
|
update_nested_dict,
|
|
42
44
|
)
|
|
43
45
|
from eodag.utils.exceptions import ValidationError
|
|
@@ -158,10 +160,6 @@ class Search(PluginTopic):
|
|
|
158
160
|
:returns: The product type definition parameters
|
|
159
161
|
"""
|
|
160
162
|
if product_type in self.config.products.keys():
|
|
161
|
-
logger.debug(
|
|
162
|
-
"Getting provider product type definition parameters for %s",
|
|
163
|
-
product_type,
|
|
164
|
-
)
|
|
165
163
|
return self.config.products[product_type]
|
|
166
164
|
elif GENERIC_PRODUCT_TYPE in self.config.products.keys():
|
|
167
165
|
logger.debug(
|
|
@@ -439,3 +437,35 @@ class Search(PluginTopic):
|
|
|
439
437
|
):
|
|
440
438
|
queryables[k] = v
|
|
441
439
|
return queryables
|
|
440
|
+
|
|
441
|
+
def get_assets_from_mapping(self, provider_item: dict[str, Any]) -> dict[str, Any]:
|
|
442
|
+
"""
|
|
443
|
+
Create assets based on the assets_mapping in the provider's config
|
|
444
|
+
and an item returned by the provider
|
|
445
|
+
|
|
446
|
+
:param provider_item: dict of item properties returned by the provider
|
|
447
|
+
:returns: dict containing the asset metadata
|
|
448
|
+
"""
|
|
449
|
+
assets_mapping = getattr(self.config, "assets_mapping", None)
|
|
450
|
+
if not assets_mapping:
|
|
451
|
+
return {}
|
|
452
|
+
assets = {}
|
|
453
|
+
for key, values in assets_mapping.items():
|
|
454
|
+
asset_href = values.get("href")
|
|
455
|
+
if not asset_href:
|
|
456
|
+
logger.warning(
|
|
457
|
+
"asset mapping %s skipped because no href is available", key
|
|
458
|
+
)
|
|
459
|
+
continue
|
|
460
|
+
json_url_path = string_to_jsonpath(asset_href)
|
|
461
|
+
if isinstance(json_url_path, str):
|
|
462
|
+
url_path = json_url_path
|
|
463
|
+
else:
|
|
464
|
+
url_match = json_url_path.find(provider_item)
|
|
465
|
+
if len(url_match) == 1:
|
|
466
|
+
url_path = url_match[0].value
|
|
467
|
+
else:
|
|
468
|
+
url_path = NOT_AVAILABLE
|
|
469
|
+
assets[key] = deepcopy(values)
|
|
470
|
+
assets[key]["href"] = url_path
|
|
471
|
+
return assets
|
|
@@ -94,6 +94,7 @@ ECMWF_KEYWORDS = {
|
|
|
94
94
|
"fcperiod",
|
|
95
95
|
"fieldset",
|
|
96
96
|
"filter",
|
|
97
|
+
"feature",
|
|
97
98
|
"format",
|
|
98
99
|
"frame",
|
|
99
100
|
"frequency",
|
|
@@ -548,7 +549,9 @@ class ECMWFSearch(PostJsonSearch):
|
|
|
548
549
|
:param params: Search parameters to be preprocessed.
|
|
549
550
|
:param product_type: (optional) product type id
|
|
550
551
|
"""
|
|
552
|
+
|
|
551
553
|
_dc_qs = params.get("_dc_qs")
|
|
554
|
+
|
|
552
555
|
if _dc_qs is not None:
|
|
553
556
|
# if available, update search params using datacube query-string
|
|
554
557
|
_dc_qp = geojson.loads(unquote_plus(unquote_plus(_dc_qs)))
|
|
@@ -284,6 +284,8 @@ class CopMarineSearch(StaticStacSearch):
|
|
|
284
284
|
"type": "application/x-netcdf",
|
|
285
285
|
}
|
|
286
286
|
}
|
|
287
|
+
additional_assets = self.get_assets_from_mapping(dataset_item)
|
|
288
|
+
assets.update(additional_assets)
|
|
287
289
|
product = EOProduct(self.provider, properties, productType=product_type)
|
|
288
290
|
# use product_type_config as default properties
|
|
289
291
|
product_type_config = getattr(self.config, "product_type_config", {})
|
|
@@ -41,6 +41,7 @@ from eodag.utils import (
|
|
|
41
41
|
GENERIC_PRODUCT_TYPE,
|
|
42
42
|
HTTP_REQ_TIMEOUT,
|
|
43
43
|
USER_AGENT,
|
|
44
|
+
_deprecated,
|
|
44
45
|
deepcopy,
|
|
45
46
|
string_to_jsonpath,
|
|
46
47
|
)
|
|
@@ -57,6 +58,10 @@ if TYPE_CHECKING:
|
|
|
57
58
|
logger = logging.getLogger("eodag.search.data_request_search")
|
|
58
59
|
|
|
59
60
|
|
|
61
|
+
@_deprecated(
|
|
62
|
+
reason="Plugin that was used in previous wekeo provider configuration, but not used anymore",
|
|
63
|
+
version="3.7.1",
|
|
64
|
+
)
|
|
60
65
|
class DataRequestSearch(Search):
|
|
61
66
|
"""
|
|
62
67
|
Plugin to execute search requests composed of several steps:
|
|
@@ -166,7 +171,7 @@ class DataRequestSearch(Search):
|
|
|
166
171
|
data_request_id: Optional[str]
|
|
167
172
|
|
|
168
173
|
def __init__(self, provider: str, config: PluginConfig) -> None:
|
|
169
|
-
super(
|
|
174
|
+
super().__init__(provider, config)
|
|
170
175
|
self.config.__dict__.setdefault("result_type", "json")
|
|
171
176
|
self.config.__dict__.setdefault("results_entry", "content")
|
|
172
177
|
self.config.__dict__.setdefault("pagination", {})
|
eodag/plugins/search/qssearch.py
CHANGED
|
@@ -35,9 +35,11 @@ from typing import (
|
|
|
35
35
|
from urllib.error import URLError
|
|
36
36
|
from urllib.parse import (
|
|
37
37
|
parse_qsl,
|
|
38
|
+
quote,
|
|
38
39
|
quote_plus,
|
|
39
40
|
unquote,
|
|
40
41
|
unquote_plus,
|
|
42
|
+
urlencode,
|
|
41
43
|
urlparse,
|
|
42
44
|
urlunparse,
|
|
43
45
|
)
|
|
@@ -86,10 +88,8 @@ from eodag.utils import (
|
|
|
86
88
|
dict_items_recursive_apply,
|
|
87
89
|
format_dict_items,
|
|
88
90
|
get_ssl_context,
|
|
89
|
-
quote,
|
|
90
91
|
string_to_jsonpath,
|
|
91
92
|
update_nested_dict,
|
|
92
|
-
urlencode,
|
|
93
93
|
)
|
|
94
94
|
from eodag.utils.exceptions import (
|
|
95
95
|
AuthenticationError,
|
|
@@ -386,36 +386,48 @@ class QueryStringSearch(Search):
|
|
|
386
386
|
|
|
387
387
|
# parse jsonpath on init: product type specific metadata-mapping
|
|
388
388
|
for product_type in self.config.products.keys():
|
|
389
|
-
|
|
390
|
-
|
|
391
|
-
|
|
392
|
-
|
|
393
|
-
|
|
394
|
-
)
|
|
389
|
+
|
|
390
|
+
product_type_metadata_mapping = {}
|
|
391
|
+
# product-type specific metadata-mapping
|
|
392
|
+
if any(
|
|
393
|
+
mm in self.config.products[product_type].keys()
|
|
394
|
+
for mm in ("metadata_mapping", "metadata_mapping_from_product")
|
|
395
|
+
):
|
|
395
396
|
# Complete and ready to use product type specific metadata-mapping
|
|
396
397
|
product_type_metadata_mapping = deepcopy(self.config.metadata_mapping)
|
|
397
398
|
|
|
398
|
-
|
|
399
|
-
|
|
400
|
-
|
|
401
|
-
|
|
402
|
-
|
|
403
|
-
|
|
404
|
-
),
|
|
399
|
+
# metadata_mapping from another product
|
|
400
|
+
if other_product_for_mapping := self.config.products[product_type].get(
|
|
401
|
+
"metadata_mapping_from_product"
|
|
402
|
+
):
|
|
403
|
+
other_product_type_def_params = self.get_product_type_def_params(
|
|
404
|
+
other_product_for_mapping,
|
|
405
405
|
)
|
|
406
|
-
|
|
407
|
-
|
|
408
|
-
|
|
409
|
-
|
|
406
|
+
# parse mapping to apply
|
|
407
|
+
if other_product_type_mtd_mapping := other_product_type_def_params.get(
|
|
408
|
+
"metadata_mapping", {}
|
|
409
|
+
):
|
|
410
410
|
other_product_type_mtd_mapping = (
|
|
411
411
|
mtd_cfg_as_conversion_and_querypath(
|
|
412
412
|
other_product_type_def_params.get("metadata_mapping", {})
|
|
413
413
|
)
|
|
414
414
|
)
|
|
415
|
-
|
|
416
|
-
|
|
417
|
-
|
|
418
|
-
|
|
415
|
+
else:
|
|
416
|
+
msg = f"Cannot reuse empty metadata_mapping from {other_product_for_mapping} for {product_type}"
|
|
417
|
+
raise MisconfiguredError(msg)
|
|
418
|
+
# update mapping
|
|
419
|
+
for metadata, mapping in other_product_type_mtd_mapping.items():
|
|
420
|
+
product_type_metadata_mapping.pop(metadata, None)
|
|
421
|
+
product_type_metadata_mapping[metadata] = mapping
|
|
422
|
+
|
|
423
|
+
# metadata_mapping from current product
|
|
424
|
+
if "metadata_mapping" in self.config.products[product_type].keys():
|
|
425
|
+
# parse mapping to apply
|
|
426
|
+
self.config.products[product_type][
|
|
427
|
+
"metadata_mapping"
|
|
428
|
+
] = mtd_cfg_as_conversion_and_querypath(
|
|
429
|
+
self.config.products[product_type]["metadata_mapping"]
|
|
430
|
+
)
|
|
419
431
|
|
|
420
432
|
# from current product, updated mapping at the end
|
|
421
433
|
for metadata, mapping in self.config.products[product_type][
|
|
@@ -424,6 +436,7 @@ class QueryStringSearch(Search):
|
|
|
424
436
|
product_type_metadata_mapping.pop(metadata, None)
|
|
425
437
|
product_type_metadata_mapping[metadata] = mapping
|
|
426
438
|
|
|
439
|
+
if product_type_metadata_mapping:
|
|
427
440
|
self.config.products[product_type][
|
|
428
441
|
"metadata_mapping"
|
|
429
442
|
] = product_type_metadata_mapping
|
|
@@ -614,6 +627,26 @@ class QueryStringSearch(Search):
|
|
|
614
627
|
generic_product_type_id
|
|
615
628
|
].update(collection_data)
|
|
616
629
|
|
|
630
|
+
# update product type id if needed
|
|
631
|
+
if collection_data_id := collection_data.get("ID"):
|
|
632
|
+
if generic_product_type_id != collection_data_id:
|
|
633
|
+
logger.debug(
|
|
634
|
+
"Rename %s product type to %s",
|
|
635
|
+
generic_product_type_id,
|
|
636
|
+
collection_data_id,
|
|
637
|
+
)
|
|
638
|
+
conf_update_dict["providers_config"][
|
|
639
|
+
collection_data_id
|
|
640
|
+
] = conf_update_dict["providers_config"].pop(
|
|
641
|
+
generic_product_type_id
|
|
642
|
+
)
|
|
643
|
+
conf_update_dict["product_types_config"][
|
|
644
|
+
collection_data_id
|
|
645
|
+
] = conf_update_dict["product_types_config"].pop(
|
|
646
|
+
generic_product_type_id
|
|
647
|
+
)
|
|
648
|
+
generic_product_type_id = collection_data_id
|
|
649
|
+
|
|
617
650
|
# update keywords
|
|
618
651
|
keywords_fields = [
|
|
619
652
|
"instrument",
|
|
@@ -1090,6 +1123,8 @@ class QueryStringSearch(Search):
|
|
|
1090
1123
|
product.properties = dict(
|
|
1091
1124
|
getattr(self.config, "product_type_config", {}), **product.properties
|
|
1092
1125
|
)
|
|
1126
|
+
additional_assets = self.get_assets_from_mapping(result)
|
|
1127
|
+
product.assets.update(additional_assets)
|
|
1093
1128
|
# move assets from properties to product's attr, normalize keys & roles
|
|
1094
1129
|
for key, asset in product.properties.pop("assets", {}).items():
|
|
1095
1130
|
norm_key, asset["roles"] = product.driver.guess_asset_key_and_roles(
|
|
@@ -1481,13 +1516,17 @@ class PostJsonSearch(QueryStringSearch):
|
|
|
1481
1516
|
)
|
|
1482
1517
|
|
|
1483
1518
|
# Add to the query, the queryable parameters set in the provider product type definition
|
|
1519
|
+
product_type_metadata_mapping = {
|
|
1520
|
+
**getattr(self.config, "metadata_mapping", {}),
|
|
1521
|
+
**prep.product_type_def_params.get("metadata_mapping", {}),
|
|
1522
|
+
}
|
|
1484
1523
|
keywords.update(
|
|
1485
1524
|
{
|
|
1486
1525
|
k: v
|
|
1487
1526
|
for k, v in prep.product_type_def_params.items()
|
|
1488
1527
|
if k not in keywords.keys()
|
|
1489
|
-
and k in
|
|
1490
|
-
and isinstance(
|
|
1528
|
+
and k in product_type_metadata_mapping.keys()
|
|
1529
|
+
and isinstance(product_type_metadata_mapping[k], list)
|
|
1491
1530
|
}
|
|
1492
1531
|
)
|
|
1493
1532
|
|