eodag 2.12.0__py3-none-any.whl → 3.0.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- eodag/__init__.py +6 -8
- eodag/api/core.py +654 -538
- eodag/api/product/__init__.py +12 -2
- eodag/api/product/_assets.py +59 -16
- eodag/api/product/_product.py +100 -93
- eodag/api/product/drivers/__init__.py +7 -2
- eodag/api/product/drivers/base.py +0 -3
- eodag/api/product/metadata_mapping.py +192 -96
- eodag/api/search_result.py +69 -10
- eodag/cli.py +55 -25
- eodag/config.py +391 -116
- eodag/plugins/apis/base.py +11 -165
- eodag/plugins/apis/ecmwf.py +36 -25
- eodag/plugins/apis/usgs.py +80 -35
- eodag/plugins/authentication/aws_auth.py +13 -4
- eodag/plugins/authentication/base.py +10 -1
- eodag/plugins/authentication/generic.py +2 -2
- eodag/plugins/authentication/header.py +31 -6
- eodag/plugins/authentication/keycloak.py +17 -84
- eodag/plugins/authentication/oauth.py +3 -3
- eodag/plugins/authentication/openid_connect.py +268 -49
- eodag/plugins/authentication/qsauth.py +4 -1
- eodag/plugins/authentication/sas_auth.py +9 -2
- eodag/plugins/authentication/token.py +98 -47
- eodag/plugins/authentication/token_exchange.py +122 -0
- eodag/plugins/crunch/base.py +3 -1
- eodag/plugins/crunch/filter_date.py +3 -9
- eodag/plugins/crunch/filter_latest_intersect.py +0 -3
- eodag/plugins/crunch/filter_latest_tpl_name.py +1 -4
- eodag/plugins/crunch/filter_overlap.py +4 -8
- eodag/plugins/crunch/filter_property.py +5 -11
- eodag/plugins/download/aws.py +149 -185
- eodag/plugins/download/base.py +88 -97
- eodag/plugins/download/creodias_s3.py +1 -1
- eodag/plugins/download/http.py +638 -310
- eodag/plugins/download/s3rest.py +47 -45
- eodag/plugins/manager.py +228 -88
- eodag/plugins/search/__init__.py +36 -0
- eodag/plugins/search/base.py +239 -30
- eodag/plugins/search/build_search_result.py +382 -37
- eodag/plugins/search/cop_marine.py +441 -0
- eodag/plugins/search/creodias_s3.py +25 -20
- eodag/plugins/search/csw.py +5 -7
- eodag/plugins/search/data_request_search.py +61 -30
- eodag/plugins/search/qssearch.py +713 -255
- eodag/plugins/search/static_stac_search.py +106 -40
- eodag/resources/ext_product_types.json +1 -1
- eodag/resources/product_types.yml +1921 -34
- eodag/resources/providers.yml +4091 -3655
- eodag/resources/stac.yml +50 -216
- eodag/resources/stac_api.yml +71 -25
- eodag/resources/stac_provider.yml +5 -0
- eodag/resources/user_conf_template.yml +89 -32
- eodag/rest/__init__.py +6 -0
- eodag/rest/cache.py +70 -0
- eodag/rest/config.py +68 -0
- eodag/rest/constants.py +26 -0
- eodag/rest/core.py +735 -0
- eodag/rest/errors.py +178 -0
- eodag/rest/server.py +264 -431
- eodag/rest/stac.py +442 -836
- eodag/rest/types/collections_search.py +44 -0
- eodag/rest/types/eodag_search.py +238 -47
- eodag/rest/types/queryables.py +164 -0
- eodag/rest/types/stac_search.py +273 -0
- eodag/rest/utils/__init__.py +216 -0
- eodag/rest/utils/cql_evaluate.py +119 -0
- eodag/rest/utils/rfc3339.py +64 -0
- eodag/types/__init__.py +106 -10
- eodag/types/bbox.py +15 -14
- eodag/types/download_args.py +40 -0
- eodag/types/search_args.py +57 -7
- eodag/types/whoosh.py +79 -0
- eodag/utils/__init__.py +110 -91
- eodag/utils/constraints.py +37 -45
- eodag/utils/exceptions.py +39 -22
- eodag/utils/import_system.py +0 -4
- eodag/utils/logging.py +37 -80
- eodag/utils/notebook.py +4 -4
- eodag/utils/repr.py +113 -0
- eodag/utils/requests.py +128 -0
- eodag/utils/rest.py +100 -0
- eodag/utils/stac_reader.py +93 -21
- {eodag-2.12.0.dist-info → eodag-3.0.0.dist-info}/METADATA +88 -53
- eodag-3.0.0.dist-info/RECORD +109 -0
- {eodag-2.12.0.dist-info → eodag-3.0.0.dist-info}/WHEEL +1 -1
- {eodag-2.12.0.dist-info → eodag-3.0.0.dist-info}/entry_points.txt +7 -5
- eodag/plugins/apis/cds.py +0 -540
- eodag/rest/types/stac_queryables.py +0 -134
- eodag/rest/utils.py +0 -1133
- eodag-2.12.0.dist-info/RECORD +0 -94
- {eodag-2.12.0.dist-info → eodag-3.0.0.dist-info}/LICENSE +0 -0
- {eodag-2.12.0.dist-info → eodag-3.0.0.dist-info}/top_level.txt +0 -0
|
@@ -26,6 +26,8 @@ from string import Formatter
|
|
|
26
26
|
from typing import (
|
|
27
27
|
TYPE_CHECKING,
|
|
28
28
|
Any,
|
|
29
|
+
AnyStr,
|
|
30
|
+
Callable,
|
|
29
31
|
Dict,
|
|
30
32
|
Iterator,
|
|
31
33
|
List,
|
|
@@ -39,8 +41,9 @@ import geojson
|
|
|
39
41
|
import orjson
|
|
40
42
|
import pyproj
|
|
41
43
|
from dateutil.parser import isoparse
|
|
44
|
+
from dateutil.relativedelta import relativedelta
|
|
42
45
|
from dateutil.tz import UTC, tzutc
|
|
43
|
-
from jsonpath_ng.jsonpath import Child
|
|
46
|
+
from jsonpath_ng.jsonpath import Child, JSONPath
|
|
44
47
|
from lxml import etree
|
|
45
48
|
from lxml.etree import XPathEvalError
|
|
46
49
|
from shapely import wkt
|
|
@@ -52,6 +55,7 @@ from eodag.utils import (
|
|
|
52
55
|
DEFAULT_PROJ,
|
|
53
56
|
deepcopy,
|
|
54
57
|
dict_items_recursive_apply,
|
|
58
|
+
format_string,
|
|
55
59
|
get_geometry_from_various,
|
|
56
60
|
get_timestamp,
|
|
57
61
|
items_recursive_apply,
|
|
@@ -79,10 +83,11 @@ OFFLINE_STATUS = "OFFLINE"
|
|
|
79
83
|
COORDS_ROUNDING_PRECISION = 4
|
|
80
84
|
WKT_MAX_LEN = 1600
|
|
81
85
|
COMPLEX_QS_REGEX = re.compile(r"^(.+=)?([^=]*)({.+})+([^=&]*)$")
|
|
86
|
+
DEFAULT_GEOMETRY = "POLYGON((180 -90, 180 90, -180 90, -180 -90, 180 -90))"
|
|
82
87
|
|
|
83
88
|
|
|
84
89
|
def get_metadata_path(
|
|
85
|
-
map_value: Union[str, List[str]]
|
|
90
|
+
map_value: Union[str, List[str]],
|
|
86
91
|
) -> Tuple[Union[List[str], None], str]:
|
|
87
92
|
"""Return the jsonpath or xpath to the value of a EO product metadata in a provider
|
|
88
93
|
search result.
|
|
@@ -116,10 +121,8 @@ def get_metadata_path(
|
|
|
116
121
|
in the provider search config. For example, it is the list
|
|
117
122
|
`['productType', '$.properties.productType']` with the sample
|
|
118
123
|
above. Or the string `$.properties.id`.
|
|
119
|
-
:type map_value: str or list(str)
|
|
120
124
|
:returns: Either, None and the path to the metadata value, or a list of converter
|
|
121
125
|
and its args, and the path to the metadata value.
|
|
122
|
-
:rtype: tuple(list(str) or None, str)
|
|
123
126
|
"""
|
|
124
127
|
path = get_metadata_path_value(map_value)
|
|
125
128
|
try:
|
|
@@ -143,16 +146,14 @@ def get_search_param(map_value: List[str]) -> str:
|
|
|
143
146
|
|
|
144
147
|
:param map_value: The value originating from the definition of `metadata_mapping`
|
|
145
148
|
in the provider search config
|
|
146
|
-
:type map_value: list
|
|
147
149
|
:returns: The value of the search parameter as defined in the provider config
|
|
148
|
-
:rtype: str
|
|
149
150
|
"""
|
|
150
151
|
# Assume that caller will pass in the value as a list
|
|
151
152
|
return map_value[0]
|
|
152
153
|
|
|
153
154
|
|
|
154
|
-
def format_metadata(search_param: str, *args:
|
|
155
|
-
"""Format a string of form {<field_name>#<conversion_function>}
|
|
155
|
+
def format_metadata(search_param: str, *args: Any, **kwargs: Any) -> str:
|
|
156
|
+
"""Format a string of form ``{<field_name>#<conversion_function>}``
|
|
156
157
|
|
|
157
158
|
The currently understood converters are:
|
|
158
159
|
- ``datetime_to_timestamp_milliseconds``: converts a utc date string to a timestamp in
|
|
@@ -186,13 +187,9 @@ def format_metadata(search_param: str, *args: Tuple[Any], **kwargs: Any) -> str:
|
|
|
186
187
|
- ``get_ecmwf_time``: get the time of a datetime string in the ECMWF format
|
|
187
188
|
|
|
188
189
|
:param search_param: The string to be formatted
|
|
189
|
-
:type search_param: str
|
|
190
190
|
:param args: (optional) Additional arguments to use in the formatting process
|
|
191
|
-
:type args: tuple
|
|
192
191
|
:param kwargs: (optional) Additional named-arguments to use when formatting
|
|
193
|
-
:type kwargs: Any
|
|
194
192
|
:returns: The formatted string
|
|
195
|
-
:rtype: str
|
|
196
193
|
"""
|
|
197
194
|
|
|
198
195
|
class MetadataFormatter(Formatter):
|
|
@@ -203,8 +200,8 @@ def format_metadata(search_param: str, *args: Tuple[Any], **kwargs: Any) -> str:
|
|
|
203
200
|
)
|
|
204
201
|
|
|
205
202
|
def __init__(self) -> None:
|
|
206
|
-
self.custom_converter = None
|
|
207
|
-
self.custom_args = None
|
|
203
|
+
self.custom_converter: Optional[Callable] = None
|
|
204
|
+
self.custom_args: Optional[str] = None
|
|
208
205
|
|
|
209
206
|
def get_field(self, field_name: str, args: Any, kwargs: Any) -> Any:
|
|
210
207
|
conversion_func_spec = self.CONVERSION_REGEX.match(field_name)
|
|
@@ -304,6 +301,11 @@ def format_metadata(search_param: str, *args: Tuple[Any], **kwargs: Any) -> str:
|
|
|
304
301
|
dt += timedelta(*time_delta_args)
|
|
305
302
|
return dt.isoformat()[:10]
|
|
306
303
|
|
|
304
|
+
@staticmethod
|
|
305
|
+
def convert_to_non_separated_date(datetime_string):
|
|
306
|
+
iso_date = MetadataFormatter.convert_to_iso_date(datetime_string)
|
|
307
|
+
return iso_date.replace("-", "")
|
|
308
|
+
|
|
307
309
|
@staticmethod
|
|
308
310
|
def convert_to_rounded_wkt(value: BaseGeometry) -> str:
|
|
309
311
|
wkt_value = cast(
|
|
@@ -379,7 +381,9 @@ def format_metadata(search_param: str, *args: Tuple[Any], **kwargs: Any) -> str:
|
|
|
379
381
|
def convert_from_ewkt(ewkt_string: str) -> Union[BaseGeometry, str]:
|
|
380
382
|
"""Convert EWKT (Extended Well-Known text) to shapely geometry"""
|
|
381
383
|
|
|
382
|
-
ewkt_regex = re.compile(
|
|
384
|
+
ewkt_regex = re.compile(
|
|
385
|
+
r"^.*(?P<proj>SRID=[0-9]+);(?P<wkt>[A-Z0-9 \(\),\.-]+).*$"
|
|
386
|
+
)
|
|
383
387
|
ewkt_match = ewkt_regex.match(ewkt_string)
|
|
384
388
|
if ewkt_match:
|
|
385
389
|
g = ewkt_match.groupdict()
|
|
@@ -462,6 +466,15 @@ def format_metadata(search_param: str, *args: Tuple[Any], **kwargs: Any) -> str:
|
|
|
462
466
|
)
|
|
463
467
|
return georss
|
|
464
468
|
|
|
469
|
+
@staticmethod
|
|
470
|
+
def convert_to_longitude_latitude(
|
|
471
|
+
input_geom_unformatted: Any,
|
|
472
|
+
) -> Dict[str, float]:
|
|
473
|
+
bounds = MetadataFormatter.convert_to_bounds(input_geom_unformatted)
|
|
474
|
+
lon = (bounds[0] + bounds[2]) / 2
|
|
475
|
+
lat = (bounds[1] + bounds[3]) / 2
|
|
476
|
+
return {"lon": lon, "lat": lat}
|
|
477
|
+
|
|
465
478
|
@staticmethod
|
|
466
479
|
def convert_csv_list(values_list: Any) -> Any:
|
|
467
480
|
if isinstance(values_list, list):
|
|
@@ -479,12 +492,15 @@ def format_metadata(search_param: str, *args: Tuple[Any], **kwargs: Any) -> str:
|
|
|
479
492
|
@staticmethod
|
|
480
493
|
def convert_get_group_name(string: str, pattern: str) -> str:
|
|
481
494
|
try:
|
|
482
|
-
|
|
495
|
+
match = re.search(pattern, str(string))
|
|
496
|
+
if match:
|
|
497
|
+
return match.lastgroup or NOT_AVAILABLE
|
|
483
498
|
except AttributeError:
|
|
484
|
-
|
|
485
|
-
|
|
486
|
-
|
|
487
|
-
|
|
499
|
+
pass
|
|
500
|
+
logger.warning(
|
|
501
|
+
"Could not extract property from %s using %s", string, pattern
|
|
502
|
+
)
|
|
503
|
+
return NOT_AVAILABLE
|
|
488
504
|
|
|
489
505
|
@staticmethod
|
|
490
506
|
def convert_replace_str(string: str, args: str) -> str:
|
|
@@ -513,10 +529,34 @@ def format_metadata(search_param: str, *args: Tuple[Any], **kwargs: Any) -> str:
|
|
|
513
529
|
|
|
514
530
|
return dict(input_dict, **new_items_dict)
|
|
515
531
|
|
|
532
|
+
@staticmethod
|
|
533
|
+
def convert_dict_filter(
|
|
534
|
+
input_dict: Dict[Any, Any], jsonpath_filter_str: str
|
|
535
|
+
) -> Dict[Any, Any]:
|
|
536
|
+
"""Fitlers dict items using jsonpath"""
|
|
537
|
+
|
|
538
|
+
jsonpath_filter = string_to_jsonpath(jsonpath_filter_str, force=True)
|
|
539
|
+
if isinstance(jsonpath_filter, str) or not isinstance(input_dict, dict):
|
|
540
|
+
return {}
|
|
541
|
+
|
|
542
|
+
keys_list = list(input_dict.keys())
|
|
543
|
+
matches = jsonpath_filter.find(input_dict)
|
|
544
|
+
result = {}
|
|
545
|
+
for match in matches:
|
|
546
|
+
# extract key index from matched jsonpath
|
|
547
|
+
matched_jsonpath_str = str(match.full_path)
|
|
548
|
+
matched_index = int(matched_jsonpath_str.split(".")[-1][1:-1])
|
|
549
|
+
key = keys_list[matched_index]
|
|
550
|
+
result[key] = match.value
|
|
551
|
+
return result
|
|
552
|
+
|
|
516
553
|
@staticmethod
|
|
517
554
|
def convert_slice_str(string: str, args: str) -> str:
|
|
518
|
-
cmin, cmax, cstep = [
|
|
519
|
-
|
|
555
|
+
cmin, cmax, cstep = [
|
|
556
|
+
int(x.strip()) if x.strip().lstrip("-").isdigit() else None
|
|
557
|
+
for x in args.split(",")
|
|
558
|
+
]
|
|
559
|
+
return string[cmin:cmax:cstep]
|
|
520
560
|
|
|
521
561
|
@staticmethod
|
|
522
562
|
def convert_fake_l2a_title_from_l1c(string: str) -> str:
|
|
@@ -595,23 +635,6 @@ def format_metadata(search_param: str, *args: Tuple[Any], **kwargs: Any) -> str:
|
|
|
595
635
|
params["polarisation"] = polarisation
|
|
596
636
|
return params
|
|
597
637
|
|
|
598
|
-
@staticmethod
|
|
599
|
-
def convert_get_processing_level_from_s1_id(product_id: str) -> str:
|
|
600
|
-
parts: List[str] = re.split(r"_(?!_)", product_id)
|
|
601
|
-
level = "LEVEL" + parts[3][0]
|
|
602
|
-
return level
|
|
603
|
-
|
|
604
|
-
@staticmethod
|
|
605
|
-
def convert_get_sensor_mode_from_s1_id(product_id: str) -> str:
|
|
606
|
-
parts: List[str] = re.split(r"_(?!_)", product_id)
|
|
607
|
-
return parts[1]
|
|
608
|
-
|
|
609
|
-
@staticmethod
|
|
610
|
-
def convert_get_processing_level_from_s2_id(product_id: str) -> str:
|
|
611
|
-
parts: List[str] = re.split(r"_(?!_)", product_id)
|
|
612
|
-
processing_level = "S2" + parts[1]
|
|
613
|
-
return processing_level
|
|
614
|
-
|
|
615
638
|
@staticmethod
|
|
616
639
|
def convert_split_id_into_s3_params(product_id: str) -> Dict[str, str]:
|
|
617
640
|
parts: List[str] = re.split(r"_(?!_)", product_id)
|
|
@@ -647,12 +670,6 @@ def format_metadata(search_param: str, *args: Tuple[Any], **kwargs: Any) -> str:
|
|
|
647
670
|
params["endDate"] = end_date.strftime("%Y-%m-%dT%H:%M:%SZ")
|
|
648
671
|
return params
|
|
649
672
|
|
|
650
|
-
@staticmethod
|
|
651
|
-
def convert_get_processing_level_from_s5p_id(product_id: str) -> str:
|
|
652
|
-
parts: List[str] = re.split(r"_(?!_)", product_id)
|
|
653
|
-
processing_level = parts[2].replace("_", "")
|
|
654
|
-
return processing_level
|
|
655
|
-
|
|
656
673
|
@staticmethod
|
|
657
674
|
def convert_split_cop_dem_id(product_id: str) -> List[int]:
|
|
658
675
|
parts = product_id.split("_")
|
|
@@ -670,17 +687,25 @@ def format_metadata(search_param: str, *args: Tuple[Any], **kwargs: Any) -> str:
|
|
|
670
687
|
return bbox
|
|
671
688
|
|
|
672
689
|
@staticmethod
|
|
673
|
-
def
|
|
674
|
-
|
|
675
|
-
|
|
676
|
-
|
|
690
|
+
def convert_dates_from_cmems_id(product_id: str):
|
|
691
|
+
date_format_1 = "[0-9]{10}"
|
|
692
|
+
date_format_2 = "[0-9]{8}"
|
|
693
|
+
dates = re.findall(date_format_1, product_id)
|
|
694
|
+
if dates:
|
|
695
|
+
date = dates[0]
|
|
696
|
+
else:
|
|
697
|
+
dates = re.findall(date_format_2, product_id)
|
|
698
|
+
date = dates[0]
|
|
699
|
+
if len(date) == 10:
|
|
700
|
+
date_time = datetime.strptime(dates[0], "%Y%m%d%H")
|
|
677
701
|
else:
|
|
678
|
-
|
|
679
|
-
|
|
680
|
-
|
|
681
|
-
|
|
682
|
-
|
|
683
|
-
|
|
702
|
+
date_time = datetime.strptime(dates[0], "%Y%m%d")
|
|
703
|
+
return {
|
|
704
|
+
"min_date": date_time.strftime("%Y-%m-%dT%H:%M:%SZ"),
|
|
705
|
+
"max_date": (date_time + timedelta(days=1)).strftime(
|
|
706
|
+
"%Y-%m-%dT%H:%M:%SZ"
|
|
707
|
+
),
|
|
708
|
+
}
|
|
684
709
|
|
|
685
710
|
@staticmethod
|
|
686
711
|
def convert_to_datetime_dict(
|
|
@@ -791,7 +816,10 @@ def format_metadata(search_param: str, *args: Tuple[Any], **kwargs: Any) -> str:
|
|
|
791
816
|
@staticmethod
|
|
792
817
|
def convert_get_dates_from_string(text: str, split_param="-"):
|
|
793
818
|
reg = "[0-9]{8}" + split_param + "[0-9]{8}"
|
|
794
|
-
|
|
819
|
+
match = re.search(reg, text)
|
|
820
|
+
if not match:
|
|
821
|
+
return NOT_AVAILABLE
|
|
822
|
+
dates_str = match.group()
|
|
795
823
|
dates = dates_str.split(split_param)
|
|
796
824
|
start_date = datetime.strptime(dates[0], "%Y%m%d")
|
|
797
825
|
end_date = datetime.strptime(dates[1], "%Y%m%d")
|
|
@@ -800,6 +828,79 @@ def format_metadata(search_param: str, *args: Tuple[Any], **kwargs: Any) -> str:
|
|
|
800
828
|
"endDate": end_date.strftime("%Y-%m-%dT%H:%M:%SZ"),
|
|
801
829
|
}
|
|
802
830
|
|
|
831
|
+
@staticmethod
|
|
832
|
+
def convert_get_hydrological_year(date: str):
|
|
833
|
+
utc_date = MetadataFormatter.convert_to_iso_utc_datetime(date)
|
|
834
|
+
date_object = datetime.strptime(utc_date, "%Y-%m-%dT%H:%M:%S.%fZ")
|
|
835
|
+
date_object_second_year = date_object + relativedelta(years=1)
|
|
836
|
+
return [
|
|
837
|
+
f'{date_object.strftime("%Y")}_{date_object_second_year.strftime("%y")}'
|
|
838
|
+
]
|
|
839
|
+
|
|
840
|
+
@staticmethod
|
|
841
|
+
def convert_get_variables_from_path(path: str):
|
|
842
|
+
if "?" not in path:
|
|
843
|
+
return []
|
|
844
|
+
variables = path.split("?")[1]
|
|
845
|
+
return variables.split(",")
|
|
846
|
+
|
|
847
|
+
@staticmethod
|
|
848
|
+
def convert_assets_list_to_dict(
|
|
849
|
+
assets_list: List[Dict[str, str]], asset_name_key: str = "title"
|
|
850
|
+
) -> Dict[str, Dict[str, str]]:
|
|
851
|
+
"""Convert a list of assets to a dictionary where keys represent
|
|
852
|
+
name of assets and are found among values of asset dictionaries.
|
|
853
|
+
|
|
854
|
+
assets_list == [
|
|
855
|
+
{"href": "foo", "title": "asset1", "name": "foo-name"},
|
|
856
|
+
{"href": "bar", "title": "path/to/asset1", "name": "bar-name"},
|
|
857
|
+
{"href": "baz", "title": "path/to/asset2", "name": "baz-name"},
|
|
858
|
+
{"href": "qux", "title": "asset3", "name": "qux-name"},
|
|
859
|
+
] and asset_name_key == "title" => {
|
|
860
|
+
"asset1": {"href": "foo", "title": "asset1", "name": "foo-name"},
|
|
861
|
+
"path/to/asset1": {"href": "bar", "title": "path/to/asset1", "name": "bar-name"},
|
|
862
|
+
"asset2": {"href": "baz", "title": "path/to/asset2", "name": "baz-name"},
|
|
863
|
+
"asset3": {"href": "qux", "title": "asset3", "name": "qux-name"},
|
|
864
|
+
}
|
|
865
|
+
assets_list == [
|
|
866
|
+
{"href": "foo", "title": "foo-title", "name": "asset1"},
|
|
867
|
+
{"href": "bar", "title": "bar-title", "name": "path/to/asset1"},
|
|
868
|
+
{"href": "baz", "title": "baz-title", "name": "path/to/asset2"},
|
|
869
|
+
{"href": "qux", "title": "qux-title", "name": "asset3"},
|
|
870
|
+
] and asset_name_key == "name" => {
|
|
871
|
+
"asset1": {"href": "foo", "title": "foo-title", "name": "asset1"},
|
|
872
|
+
"path/to/asset1": {"href": "bar", "title": "bar-title", "name": "path/to/asset1"},
|
|
873
|
+
"asset2": {"href": "baz", "title": "baz-title", "name": "path/to/asset2"},
|
|
874
|
+
"asset3": {"href": "qux", "title": "qux-title", "name": "asset3"},
|
|
875
|
+
}
|
|
876
|
+
"""
|
|
877
|
+
asset_names: List[str] = []
|
|
878
|
+
assets_dict: Dict[str, Dict[str, str]] = {}
|
|
879
|
+
|
|
880
|
+
for asset in assets_list:
|
|
881
|
+
asset_name = asset[asset_name_key]
|
|
882
|
+
asset_names.append(asset_name)
|
|
883
|
+
assets_dict[asset_name] = asset
|
|
884
|
+
|
|
885
|
+
# we only keep the equivalent of the path basename in the case where the
|
|
886
|
+
# asset name has a path pattern and this basename is only found once
|
|
887
|
+
immutable_asset_indexes: List[int] = []
|
|
888
|
+
for i, asset_name in enumerate(asset_names):
|
|
889
|
+
if i in immutable_asset_indexes:
|
|
890
|
+
continue
|
|
891
|
+
change_asset_name = True
|
|
892
|
+
asset_basename = asset_name.split("/")[-1]
|
|
893
|
+
j = i + 1
|
|
894
|
+
while change_asset_name and j < len(asset_names):
|
|
895
|
+
asset_tmp_basename = asset_names[j].split("/")[-1]
|
|
896
|
+
if asset_basename == asset_tmp_basename:
|
|
897
|
+
change_asset_name = False
|
|
898
|
+
immutable_asset_indexes.extend([i, j])
|
|
899
|
+
j += 1
|
|
900
|
+
if change_asset_name:
|
|
901
|
+
assets_dict[asset_basename] = assets_dict.pop(asset_name)
|
|
902
|
+
return assets_dict
|
|
903
|
+
|
|
803
904
|
# if stac extension colon separator `:` is in search params, parse it to prevent issues with vformat
|
|
804
905
|
if re.search(r"{[a-zA-Z0-9_-]*:[a-zA-Z0-9_-]*}", search_param):
|
|
805
906
|
search_param = re.sub(
|
|
@@ -818,7 +919,6 @@ def properties_from_json(
|
|
|
818
919
|
"""Extract properties from a provider json result.
|
|
819
920
|
|
|
820
921
|
:param json: The representation of a provider result as a json object
|
|
821
|
-
:type json: dict
|
|
822
922
|
:param mapping: A mapping between :class:`~eodag.api.product._product.EOProduct`'s metadata
|
|
823
923
|
keys and the location of the values of these properties in the json
|
|
824
924
|
representation, expressed as a
|
|
@@ -826,9 +926,7 @@ def properties_from_json(
|
|
|
826
926
|
:param discovery_config: (optional) metadata discovery configuration dict, accepting among other items
|
|
827
927
|
`discovery_pattern` (Regex pattern for metadata key discovery, e.g. "^[a-zA-Z]+$"),
|
|
828
928
|
`discovery_path` (String representation of jsonpath)
|
|
829
|
-
:type discovery_config: dict
|
|
830
929
|
:returns: The metadata of the :class:`~eodag.api.product._product.EOProduct`
|
|
831
|
-
:rtype: dict
|
|
832
930
|
"""
|
|
833
931
|
properties: Dict[str, Any] = {}
|
|
834
932
|
templates = {}
|
|
@@ -840,7 +938,7 @@ def properties_from_json(
|
|
|
840
938
|
else:
|
|
841
939
|
conversion_or_none, path_or_text = value
|
|
842
940
|
if isinstance(path_or_text, str):
|
|
843
|
-
if re.search(r"({[^{}]+})+", path_or_text):
|
|
941
|
+
if re.search(r"({[^{}:]+})+", path_or_text):
|
|
844
942
|
templates[metadata] = path_or_text
|
|
845
943
|
else:
|
|
846
944
|
properties[metadata] = path_or_text
|
|
@@ -874,7 +972,7 @@ def properties_from_json(
|
|
|
874
972
|
conversion_or_none = conversion_or_none[0]
|
|
875
973
|
|
|
876
974
|
# check if conversion uses variables to format
|
|
877
|
-
if re.search(r"({[^{}]+})+", conversion_or_none):
|
|
975
|
+
if re.search(r"({[^{}:]+})+", conversion_or_none):
|
|
878
976
|
conversion_or_none = conversion_or_none.format(**properties)
|
|
879
977
|
|
|
880
978
|
properties[metadata] = format_metadata(
|
|
@@ -890,7 +988,7 @@ def properties_from_json(
|
|
|
890
988
|
# Resolve templates
|
|
891
989
|
for metadata, template in templates.items():
|
|
892
990
|
try:
|
|
893
|
-
properties[metadata] = template
|
|
991
|
+
properties[metadata] = format_string(metadata, template, **properties)
|
|
894
992
|
except ValueError:
|
|
895
993
|
logger.warning(
|
|
896
994
|
f"Could not parse {metadata} ({template}) using product properties"
|
|
@@ -905,13 +1003,18 @@ def properties_from_json(
|
|
|
905
1003
|
discovery_pattern = discovery_config.get("metadata_pattern", None)
|
|
906
1004
|
discovery_path = discovery_config.get("metadata_path", None)
|
|
907
1005
|
if discovery_pattern and discovery_path:
|
|
908
|
-
|
|
1006
|
+
discovery_jsonpath = string_to_jsonpath(discovery_path)
|
|
1007
|
+
discovered_properties = (
|
|
1008
|
+
discovery_jsonpath.find(json)
|
|
1009
|
+
if isinstance(discovery_jsonpath, JSONPath)
|
|
1010
|
+
else []
|
|
1011
|
+
)
|
|
909
1012
|
for found_jsonpath in discovered_properties:
|
|
910
1013
|
if "metadata_path_id" in discovery_config.keys():
|
|
911
1014
|
found_key_paths = string_to_jsonpath(
|
|
912
1015
|
discovery_config["metadata_path_id"], force=True
|
|
913
1016
|
).find(found_jsonpath.value)
|
|
914
|
-
if not found_key_paths:
|
|
1017
|
+
if not found_key_paths or isinstance(found_key_paths, int):
|
|
915
1018
|
continue
|
|
916
1019
|
found_key = found_key_paths[0].value
|
|
917
1020
|
used_jsonpath = Child(
|
|
@@ -934,7 +1037,9 @@ def properties_from_json(
|
|
|
934
1037
|
discovery_config["metadata_path_value"], force=True
|
|
935
1038
|
).find(found_jsonpath.value)
|
|
936
1039
|
properties[found_key] = (
|
|
937
|
-
found_value_path[0].value
|
|
1040
|
+
found_value_path[0].value
|
|
1041
|
+
if found_value_path and not isinstance(found_value_path, int)
|
|
1042
|
+
else NOT_AVAILABLE
|
|
938
1043
|
)
|
|
939
1044
|
else:
|
|
940
1045
|
# default value got from metadata_path
|
|
@@ -950,7 +1055,7 @@ def properties_from_json(
|
|
|
950
1055
|
|
|
951
1056
|
|
|
952
1057
|
def properties_from_xml(
|
|
953
|
-
xml_as_text:
|
|
1058
|
+
xml_as_text: AnyStr,
|
|
954
1059
|
mapping: Any,
|
|
955
1060
|
empty_ns_prefix: str = "ns",
|
|
956
1061
|
discovery_config: Optional[Dict[str, Any]] = None,
|
|
@@ -958,7 +1063,6 @@ def properties_from_xml(
|
|
|
958
1063
|
"""Extract properties from a provider xml result.
|
|
959
1064
|
|
|
960
1065
|
:param xml_as_text: The representation of a provider result as xml
|
|
961
|
-
:type xml_as_text: str
|
|
962
1066
|
:param mapping: A mapping between :class:`~eodag.api.product._product.EOProduct`'s metadata
|
|
963
1067
|
keys and the location of the values of these properties in the xml
|
|
964
1068
|
representation, expressed as a
|
|
@@ -968,13 +1072,10 @@ def properties_from_xml(
|
|
|
968
1072
|
not supporting empty namespace prefix. The
|
|
969
1073
|
xpath in `mapping` must use this value to be able to
|
|
970
1074
|
correctly reach empty-namespace prefixed elements
|
|
971
|
-
:type empty_ns_prefix: str
|
|
972
1075
|
:param discovery_config: (optional) metadata discovery configuration dict, accepting among other items
|
|
973
1076
|
`discovery_pattern` (Regex pattern for metadata key discovery, e.g. "^[a-zA-Z]+$"),
|
|
974
1077
|
`discovery_path` (String representation of xpath)
|
|
975
|
-
:type discovery_config: dict
|
|
976
1078
|
:returns: the metadata of the :class:`~eodag.api.product._product.EOProduct`
|
|
977
|
-
:rtype: dict
|
|
978
1079
|
"""
|
|
979
1080
|
properties: Dict[str, Any] = {}
|
|
980
1081
|
templates = {}
|
|
@@ -1051,7 +1152,7 @@ def properties_from_xml(
|
|
|
1051
1152
|
conversion_or_none = conversion_or_none[0]
|
|
1052
1153
|
|
|
1053
1154
|
# check if conversion uses variables to format
|
|
1054
|
-
if re.search(r"({[^{}]+})+", conversion_or_none):
|
|
1155
|
+
if re.search(r"({[^{}:]+})+", conversion_or_none):
|
|
1055
1156
|
conversion_or_none = conversion_or_none.format(**properties)
|
|
1056
1157
|
|
|
1057
1158
|
properties[metadata] = [
|
|
@@ -1073,7 +1174,7 @@ def properties_from_xml(
|
|
|
1073
1174
|
# formatting resolution using previously successfully resolved properties
|
|
1074
1175
|
# Ignore any transformation specified. If a value is to be passed as is,
|
|
1075
1176
|
# we don't want to transform it further
|
|
1076
|
-
if re.search(r"({[^{}]+})+", path_or_text):
|
|
1177
|
+
if re.search(r"({[^{}:]+})+", path_or_text):
|
|
1077
1178
|
templates[metadata] = path_or_text
|
|
1078
1179
|
else:
|
|
1079
1180
|
properties[metadata] = path_or_text
|
|
@@ -1108,16 +1209,13 @@ def mtd_cfg_as_conversion_and_querypath(
|
|
|
1108
1209
|
dest_dict: Dict[str, Any] = {},
|
|
1109
1210
|
result_type: str = "json",
|
|
1110
1211
|
) -> Dict[str, Any]:
|
|
1111
|
-
"""Metadata configuration dictionary to querypath with conversion
|
|
1212
|
+
"""Metadata configuration dictionary to querypath with conversion dictionary
|
|
1112
1213
|
Transform every src_dict value from jsonpath_str to tuple `(conversion, jsonpath_object)`
|
|
1113
1214
|
or from xpath_str to tuple `(conversion, xpath_str)`
|
|
1114
1215
|
|
|
1115
1216
|
:param src_dict: Input dict containing jsonpath str as values
|
|
1116
|
-
:type src_dict: dict
|
|
1117
1217
|
:param dest_dict: (optional) Output dict containing jsonpath objects as values
|
|
1118
|
-
:type dest_dict: dict
|
|
1119
1218
|
:returns: dest_dict
|
|
1120
|
-
:rtype: dict
|
|
1121
1219
|
"""
|
|
1122
1220
|
# check if the configuration has already been converted
|
|
1123
1221
|
some_configured_value = (
|
|
@@ -1146,7 +1244,7 @@ def mtd_cfg_as_conversion_and_querypath(
|
|
|
1146
1244
|
else:
|
|
1147
1245
|
parsed_path = path
|
|
1148
1246
|
|
|
1149
|
-
if len(dest_dict[metadata]) == 2:
|
|
1247
|
+
if isinstance(dest_dict[metadata], list) and len(dest_dict[metadata]) == 2:
|
|
1150
1248
|
dest_dict[metadata][1] = (conversion, parsed_path)
|
|
1151
1249
|
else:
|
|
1152
1250
|
dest_dict[metadata] = (conversion, parsed_path)
|
|
@@ -1158,13 +1256,13 @@ def mtd_cfg_as_conversion_and_querypath(
|
|
|
1158
1256
|
|
|
1159
1257
|
|
|
1160
1258
|
def format_query_params(
|
|
1161
|
-
product_type: str, config: PluginConfig,
|
|
1259
|
+
product_type: str, config: PluginConfig, query_dict: Dict[str, Any]
|
|
1162
1260
|
) -> Dict[str, Any]:
|
|
1163
1261
|
"""format the search parameters to query parameters"""
|
|
1164
|
-
if "raise_errors" in
|
|
1165
|
-
del
|
|
1262
|
+
if "raise_errors" in query_dict.keys():
|
|
1263
|
+
del query_dict["raise_errors"]
|
|
1166
1264
|
# . not allowed in eodag_search_key, replaced with %2E
|
|
1167
|
-
|
|
1265
|
+
query_dict = {k.replace(".", "%2E"): v for k, v in query_dict.items()}
|
|
1168
1266
|
|
|
1169
1267
|
product_type_metadata_mapping = dict(
|
|
1170
1268
|
config.metadata_mapping,
|
|
@@ -1174,16 +1272,16 @@ def format_query_params(
|
|
|
1174
1272
|
query_params: Dict[str, Any] = {}
|
|
1175
1273
|
# Get all the search parameters that are recognised as queryables by the
|
|
1176
1274
|
# provider (they appear in the queryables dictionary)
|
|
1177
|
-
queryables = _get_queryables(
|
|
1275
|
+
queryables = _get_queryables(query_dict, config, product_type_metadata_mapping)
|
|
1178
1276
|
|
|
1179
1277
|
for eodag_search_key, provider_search_key in queryables.items():
|
|
1180
|
-
user_input =
|
|
1278
|
+
user_input = query_dict[eodag_search_key]
|
|
1181
1279
|
|
|
1182
1280
|
if COMPLEX_QS_REGEX.match(provider_search_key):
|
|
1183
1281
|
parts = provider_search_key.split("=")
|
|
1184
1282
|
if len(parts) == 1:
|
|
1185
1283
|
formatted_query_param = format_metadata(
|
|
1186
|
-
provider_search_key, product_type, **
|
|
1284
|
+
provider_search_key, product_type, **query_dict
|
|
1187
1285
|
)
|
|
1188
1286
|
formatted_query_param = formatted_query_param.replace("'", '"')
|
|
1189
1287
|
if "{{" in provider_search_key:
|
|
@@ -1202,7 +1300,7 @@ def format_query_params(
|
|
|
1202
1300
|
else:
|
|
1203
1301
|
provider_search_key, provider_value = parts
|
|
1204
1302
|
query_params.setdefault(provider_search_key, []).append(
|
|
1205
|
-
format_metadata(provider_value, product_type, **
|
|
1303
|
+
format_metadata(provider_value, product_type, **query_dict)
|
|
1206
1304
|
)
|
|
1207
1305
|
else:
|
|
1208
1306
|
query_params[provider_search_key] = user_input
|
|
@@ -1221,7 +1319,7 @@ def format_query_params(
|
|
|
1221
1319
|
**config.products.get(product_type, {}).get("metadata_mapping", {}),
|
|
1222
1320
|
)
|
|
1223
1321
|
literal_search_params.update(
|
|
1224
|
-
_format_free_text_search(config, product_type_metadata_mapping, **
|
|
1322
|
+
_format_free_text_search(config, product_type_metadata_mapping, **query_dict)
|
|
1225
1323
|
)
|
|
1226
1324
|
for provider_search_key, provider_value in literal_search_params.items():
|
|
1227
1325
|
if isinstance(provider_value, list):
|
|
@@ -1361,13 +1459,18 @@ def get_queryable_from_provider(
|
|
|
1361
1459
|
"""Get EODAG configured queryable parameter from provider queryable parameter
|
|
1362
1460
|
|
|
1363
1461
|
:param provider_queryable: provider queryable parameter
|
|
1364
|
-
:type provider_queryable: str
|
|
1365
1462
|
:param metadata_mapping: metadata-mapping configuration
|
|
1366
|
-
:type metadata_mapping: Dict[str, Union[str, List[str]]])
|
|
1367
1463
|
:returns: EODAG configured queryable parameter or None
|
|
1368
|
-
:rtype: Optional[str]
|
|
1369
1464
|
"""
|
|
1370
1465
|
pattern = rf"\b{provider_queryable}\b"
|
|
1466
|
+
# if 1:1 mapping exists privilege this one instead of other mapping
|
|
1467
|
+
# e.g. provider queryable = year -> use year and not date in which year also appears
|
|
1468
|
+
mapping_values = [
|
|
1469
|
+
v[0] if isinstance(v, list) else "" for v in metadata_mapping.values()
|
|
1470
|
+
]
|
|
1471
|
+
if provider_queryable in mapping_values:
|
|
1472
|
+
ind = mapping_values.index(provider_queryable)
|
|
1473
|
+
return Queryables.get_queryable_from_alias(list(metadata_mapping.keys())[ind])
|
|
1371
1474
|
for param, param_conf in metadata_mapping.items():
|
|
1372
1475
|
if isinstance(param_conf, list) and re.search(pattern, param_conf[0]):
|
|
1373
1476
|
return Queryables.get_queryable_from_alias(param)
|
|
@@ -1380,11 +1483,8 @@ def get_provider_queryable_path(
|
|
|
1380
1483
|
"""Get EODAG configured queryable path from its parameter
|
|
1381
1484
|
|
|
1382
1485
|
:param queryable: eodag queryable parameter
|
|
1383
|
-
:type queryable: str
|
|
1384
1486
|
:param metadata_mapping: metadata-mapping configuration
|
|
1385
|
-
:type metadata_mapping: Dict[str, Union[str, List[str]]])
|
|
1386
1487
|
:returns: EODAG configured queryable path or None
|
|
1387
|
-
:rtype: Optional[str]
|
|
1388
1488
|
"""
|
|
1389
1489
|
parameter_conf = metadata_mapping.get(queryable, None)
|
|
1390
1490
|
if isinstance(parameter_conf, list):
|
|
@@ -1400,13 +1500,9 @@ def get_provider_queryable_key(
|
|
|
1400
1500
|
) -> str:
|
|
1401
1501
|
"""finds the provider queryable corresponding to the given eodag key based on the metadata mapping
|
|
1402
1502
|
:param eodag_key: key in eodag
|
|
1403
|
-
:type eodag_key: str
|
|
1404
1503
|
:param provider_queryables: queryables returned from the provider
|
|
1405
|
-
:type provider_queryables: dict
|
|
1406
1504
|
:param metadata_mapping: metadata mapping from which the keys are retrieved
|
|
1407
|
-
:type metadata_mapping: Dict[str, Union[List[Any], str]]
|
|
1408
1505
|
:returns: provider queryable key
|
|
1409
|
-
:rtype: str
|
|
1410
1506
|
"""
|
|
1411
1507
|
if eodag_key not in metadata_mapping:
|
|
1412
1508
|
return ""
|