eodag 2.12.1__py3-none-any.whl → 3.0.0b2__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- eodag/api/core.py +440 -321
- eodag/api/product/__init__.py +5 -1
- eodag/api/product/_assets.py +57 -2
- eodag/api/product/_product.py +89 -68
- eodag/api/product/metadata_mapping.py +181 -66
- eodag/api/search_result.py +48 -1
- eodag/cli.py +20 -6
- eodag/config.py +95 -6
- eodag/plugins/apis/base.py +8 -165
- eodag/plugins/apis/ecmwf.py +36 -24
- eodag/plugins/apis/usgs.py +40 -24
- eodag/plugins/authentication/aws_auth.py +2 -2
- eodag/plugins/authentication/header.py +31 -6
- eodag/plugins/authentication/keycloak.py +13 -84
- eodag/plugins/authentication/oauth.py +3 -3
- eodag/plugins/authentication/openid_connect.py +256 -46
- eodag/plugins/authentication/qsauth.py +3 -0
- eodag/plugins/authentication/sas_auth.py +8 -1
- eodag/plugins/authentication/token.py +92 -46
- eodag/plugins/authentication/token_exchange.py +120 -0
- eodag/plugins/download/aws.py +86 -91
- eodag/plugins/download/base.py +72 -40
- eodag/plugins/download/http.py +607 -264
- eodag/plugins/download/s3rest.py +28 -15
- eodag/plugins/manager.py +74 -57
- eodag/plugins/search/__init__.py +36 -0
- eodag/plugins/search/base.py +225 -18
- eodag/plugins/search/build_search_result.py +389 -32
- eodag/plugins/search/cop_marine.py +378 -0
- eodag/plugins/search/creodias_s3.py +15 -14
- eodag/plugins/search/csw.py +5 -7
- eodag/plugins/search/data_request_search.py +44 -20
- eodag/plugins/search/qssearch.py +508 -203
- eodag/plugins/search/static_stac_search.py +99 -36
- eodag/resources/constraints/climate-dt.json +13 -0
- eodag/resources/constraints/extremes-dt.json +8 -0
- eodag/resources/ext_product_types.json +1 -1
- eodag/resources/product_types.yml +1897 -34
- eodag/resources/providers.yml +3539 -3277
- eodag/resources/stac.yml +48 -54
- eodag/resources/stac_api.yml +71 -25
- eodag/resources/stac_provider.yml +5 -0
- eodag/resources/user_conf_template.yml +51 -3
- eodag/rest/__init__.py +6 -0
- eodag/rest/cache.py +70 -0
- eodag/rest/config.py +68 -0
- eodag/rest/constants.py +27 -0
- eodag/rest/core.py +757 -0
- eodag/rest/server.py +397 -258
- eodag/rest/stac.py +438 -307
- eodag/rest/types/collections_search.py +44 -0
- eodag/rest/types/eodag_search.py +232 -43
- eodag/rest/types/{stac_queryables.py → queryables.py} +81 -43
- eodag/rest/types/stac_search.py +277 -0
- eodag/rest/utils/__init__.py +216 -0
- eodag/rest/utils/cql_evaluate.py +119 -0
- eodag/rest/utils/rfc3339.py +65 -0
- eodag/types/__init__.py +99 -9
- eodag/types/bbox.py +15 -14
- eodag/types/download_args.py +31 -0
- eodag/types/search_args.py +58 -7
- eodag/types/whoosh.py +81 -0
- eodag/utils/__init__.py +72 -9
- eodag/utils/constraints.py +37 -37
- eodag/utils/exceptions.py +23 -17
- eodag/utils/repr.py +113 -0
- eodag/utils/requests.py +138 -0
- eodag/utils/rest.py +104 -0
- eodag/utils/stac_reader.py +100 -16
- {eodag-2.12.1.dist-info → eodag-3.0.0b2.dist-info}/METADATA +65 -44
- eodag-3.0.0b2.dist-info/RECORD +110 -0
- {eodag-2.12.1.dist-info → eodag-3.0.0b2.dist-info}/WHEEL +1 -1
- {eodag-2.12.1.dist-info → eodag-3.0.0b2.dist-info}/entry_points.txt +6 -5
- eodag/plugins/apis/cds.py +0 -540
- eodag/rest/utils.py +0 -1133
- eodag-2.12.1.dist-info/RECORD +0 -94
- {eodag-2.12.1.dist-info → eodag-3.0.0b2.dist-info}/LICENSE +0 -0
- {eodag-2.12.1.dist-info → eodag-3.0.0b2.dist-info}/top_level.txt +0 -0
|
@@ -26,6 +26,8 @@ from string import Formatter
|
|
|
26
26
|
from typing import (
|
|
27
27
|
TYPE_CHECKING,
|
|
28
28
|
Any,
|
|
29
|
+
AnyStr,
|
|
30
|
+
Callable,
|
|
29
31
|
Dict,
|
|
30
32
|
Iterator,
|
|
31
33
|
List,
|
|
@@ -40,7 +42,7 @@ import orjson
|
|
|
40
42
|
import pyproj
|
|
41
43
|
from dateutil.parser import isoparse
|
|
42
44
|
from dateutil.tz import UTC, tzutc
|
|
43
|
-
from jsonpath_ng.jsonpath import Child
|
|
45
|
+
from jsonpath_ng.jsonpath import Child, JSONPath
|
|
44
46
|
from lxml import etree
|
|
45
47
|
from lxml.etree import XPathEvalError
|
|
46
48
|
from shapely import wkt
|
|
@@ -52,6 +54,7 @@ from eodag.utils import (
|
|
|
52
54
|
DEFAULT_PROJ,
|
|
53
55
|
deepcopy,
|
|
54
56
|
dict_items_recursive_apply,
|
|
57
|
+
format_string,
|
|
55
58
|
get_geometry_from_various,
|
|
56
59
|
get_timestamp,
|
|
57
60
|
items_recursive_apply,
|
|
@@ -79,10 +82,11 @@ OFFLINE_STATUS = "OFFLINE"
|
|
|
79
82
|
COORDS_ROUNDING_PRECISION = 4
|
|
80
83
|
WKT_MAX_LEN = 1600
|
|
81
84
|
COMPLEX_QS_REGEX = re.compile(r"^(.+=)?([^=]*)({.+})+([^=&]*)$")
|
|
85
|
+
DEFAULT_GEOMETRY = "POLYGON((180 -90, 180 90, -180 90, -180 -90, 180 -90))"
|
|
82
86
|
|
|
83
87
|
|
|
84
88
|
def get_metadata_path(
|
|
85
|
-
map_value: Union[str, List[str]]
|
|
89
|
+
map_value: Union[str, List[str]],
|
|
86
90
|
) -> Tuple[Union[List[str], None], str]:
|
|
87
91
|
"""Return the jsonpath or xpath to the value of a EO product metadata in a provider
|
|
88
92
|
search result.
|
|
@@ -151,7 +155,7 @@ def get_search_param(map_value: List[str]) -> str:
|
|
|
151
155
|
return map_value[0]
|
|
152
156
|
|
|
153
157
|
|
|
154
|
-
def format_metadata(search_param: str, *args:
|
|
158
|
+
def format_metadata(search_param: str, *args: Any, **kwargs: Any) -> str:
|
|
155
159
|
"""Format a string of form {<field_name>#<conversion_function>}
|
|
156
160
|
|
|
157
161
|
The currently understood converters are:
|
|
@@ -203,8 +207,8 @@ def format_metadata(search_param: str, *args: Tuple[Any], **kwargs: Any) -> str:
|
|
|
203
207
|
)
|
|
204
208
|
|
|
205
209
|
def __init__(self) -> None:
|
|
206
|
-
self.custom_converter = None
|
|
207
|
-
self.custom_args = None
|
|
210
|
+
self.custom_converter: Optional[Callable] = None
|
|
211
|
+
self.custom_args: Optional[str] = None
|
|
208
212
|
|
|
209
213
|
def get_field(self, field_name: str, args: Any, kwargs: Any) -> Any:
|
|
210
214
|
conversion_func_spec = self.CONVERSION_REGEX.match(field_name)
|
|
@@ -304,6 +308,11 @@ def format_metadata(search_param: str, *args: Tuple[Any], **kwargs: Any) -> str:
|
|
|
304
308
|
dt += timedelta(*time_delta_args)
|
|
305
309
|
return dt.isoformat()[:10]
|
|
306
310
|
|
|
311
|
+
@staticmethod
|
|
312
|
+
def convert_to_non_separated_date(datetime_string):
|
|
313
|
+
iso_date = MetadataFormatter.convert_to_iso_date(datetime_string)
|
|
314
|
+
return iso_date.replace("-", "")
|
|
315
|
+
|
|
307
316
|
@staticmethod
|
|
308
317
|
def convert_to_rounded_wkt(value: BaseGeometry) -> str:
|
|
309
318
|
wkt_value = cast(
|
|
@@ -379,7 +388,9 @@ def format_metadata(search_param: str, *args: Tuple[Any], **kwargs: Any) -> str:
|
|
|
379
388
|
def convert_from_ewkt(ewkt_string: str) -> Union[BaseGeometry, str]:
|
|
380
389
|
"""Convert EWKT (Extended Well-Known text) to shapely geometry"""
|
|
381
390
|
|
|
382
|
-
ewkt_regex = re.compile(
|
|
391
|
+
ewkt_regex = re.compile(
|
|
392
|
+
r"^.*(?P<proj>SRID=[0-9]+);(?P<wkt>[A-Z0-9 \(\),\.-]+).*$"
|
|
393
|
+
)
|
|
383
394
|
ewkt_match = ewkt_regex.match(ewkt_string)
|
|
384
395
|
if ewkt_match:
|
|
385
396
|
g = ewkt_match.groupdict()
|
|
@@ -462,6 +473,15 @@ def format_metadata(search_param: str, *args: Tuple[Any], **kwargs: Any) -> str:
|
|
|
462
473
|
)
|
|
463
474
|
return georss
|
|
464
475
|
|
|
476
|
+
@staticmethod
|
|
477
|
+
def convert_to_longitude_latitude(
|
|
478
|
+
input_geom_unformatted: Any,
|
|
479
|
+
) -> Dict[str, float]:
|
|
480
|
+
bounds = MetadataFormatter.convert_to_bounds(input_geom_unformatted)
|
|
481
|
+
lon = (bounds[0] + bounds[2]) / 2
|
|
482
|
+
lat = (bounds[1] + bounds[3]) / 2
|
|
483
|
+
return {"lon": lon, "lat": lat}
|
|
484
|
+
|
|
465
485
|
@staticmethod
|
|
466
486
|
def convert_csv_list(values_list: Any) -> Any:
|
|
467
487
|
if isinstance(values_list, list):
|
|
@@ -479,12 +499,15 @@ def format_metadata(search_param: str, *args: Tuple[Any], **kwargs: Any) -> str:
|
|
|
479
499
|
@staticmethod
|
|
480
500
|
def convert_get_group_name(string: str, pattern: str) -> str:
|
|
481
501
|
try:
|
|
482
|
-
|
|
502
|
+
match = re.search(pattern, str(string))
|
|
503
|
+
if match:
|
|
504
|
+
return match.lastgroup or NOT_AVAILABLE
|
|
483
505
|
except AttributeError:
|
|
484
|
-
|
|
485
|
-
|
|
486
|
-
|
|
487
|
-
|
|
506
|
+
pass
|
|
507
|
+
logger.warning(
|
|
508
|
+
"Could not extract property from %s using %s", string, pattern
|
|
509
|
+
)
|
|
510
|
+
return NOT_AVAILABLE
|
|
488
511
|
|
|
489
512
|
@staticmethod
|
|
490
513
|
def convert_replace_str(string: str, args: str) -> str:
|
|
@@ -513,10 +536,34 @@ def format_metadata(search_param: str, *args: Tuple[Any], **kwargs: Any) -> str:
|
|
|
513
536
|
|
|
514
537
|
return dict(input_dict, **new_items_dict)
|
|
515
538
|
|
|
539
|
+
@staticmethod
|
|
540
|
+
def convert_dict_filter(
|
|
541
|
+
input_dict: Dict[Any, Any], jsonpath_filter_str: str
|
|
542
|
+
) -> Dict[Any, Any]:
|
|
543
|
+
"""Fitlers dict items using jsonpath"""
|
|
544
|
+
|
|
545
|
+
jsonpath_filter = string_to_jsonpath(jsonpath_filter_str, force=True)
|
|
546
|
+
if isinstance(jsonpath_filter, str) or not isinstance(input_dict, dict):
|
|
547
|
+
return {}
|
|
548
|
+
|
|
549
|
+
keys_list = list(input_dict.keys())
|
|
550
|
+
matches = jsonpath_filter.find(input_dict)
|
|
551
|
+
result = {}
|
|
552
|
+
for match in matches:
|
|
553
|
+
# extract key index from matched jsonpath
|
|
554
|
+
matched_jsonpath_str = str(match.full_path)
|
|
555
|
+
matched_index = int(matched_jsonpath_str.split(".")[-1][1:-1])
|
|
556
|
+
key = keys_list[matched_index]
|
|
557
|
+
result[key] = match.value
|
|
558
|
+
return result
|
|
559
|
+
|
|
516
560
|
@staticmethod
|
|
517
561
|
def convert_slice_str(string: str, args: str) -> str:
|
|
518
|
-
cmin, cmax, cstep = [
|
|
519
|
-
|
|
562
|
+
cmin, cmax, cstep = [
|
|
563
|
+
int(x.strip()) if x.strip().lstrip("-").isdigit() else None
|
|
564
|
+
for x in args.split(",")
|
|
565
|
+
]
|
|
566
|
+
return string[cmin:cmax:cstep]
|
|
520
567
|
|
|
521
568
|
@staticmethod
|
|
522
569
|
def convert_fake_l2a_title_from_l1c(string: str) -> str:
|
|
@@ -595,23 +642,6 @@ def format_metadata(search_param: str, *args: Tuple[Any], **kwargs: Any) -> str:
|
|
|
595
642
|
params["polarisation"] = polarisation
|
|
596
643
|
return params
|
|
597
644
|
|
|
598
|
-
@staticmethod
|
|
599
|
-
def convert_get_processing_level_from_s1_id(product_id: str) -> str:
|
|
600
|
-
parts: List[str] = re.split(r"_(?!_)", product_id)
|
|
601
|
-
level = "LEVEL" + parts[3][0]
|
|
602
|
-
return level
|
|
603
|
-
|
|
604
|
-
@staticmethod
|
|
605
|
-
def convert_get_sensor_mode_from_s1_id(product_id: str) -> str:
|
|
606
|
-
parts: List[str] = re.split(r"_(?!_)", product_id)
|
|
607
|
-
return parts[1]
|
|
608
|
-
|
|
609
|
-
@staticmethod
|
|
610
|
-
def convert_get_processing_level_from_s2_id(product_id: str) -> str:
|
|
611
|
-
parts: List[str] = re.split(r"_(?!_)", product_id)
|
|
612
|
-
processing_level = "S2" + parts[1]
|
|
613
|
-
return processing_level
|
|
614
|
-
|
|
615
645
|
@staticmethod
|
|
616
646
|
def convert_split_id_into_s3_params(product_id: str) -> Dict[str, str]:
|
|
617
647
|
parts: List[str] = re.split(r"_(?!_)", product_id)
|
|
@@ -647,12 +677,6 @@ def format_metadata(search_param: str, *args: Tuple[Any], **kwargs: Any) -> str:
|
|
|
647
677
|
params["endDate"] = end_date.strftime("%Y-%m-%dT%H:%M:%SZ")
|
|
648
678
|
return params
|
|
649
679
|
|
|
650
|
-
@staticmethod
|
|
651
|
-
def convert_get_processing_level_from_s5p_id(product_id: str) -> str:
|
|
652
|
-
parts: List[str] = re.split(r"_(?!_)", product_id)
|
|
653
|
-
processing_level = parts[2].replace("_", "")
|
|
654
|
-
return processing_level
|
|
655
|
-
|
|
656
680
|
@staticmethod
|
|
657
681
|
def convert_split_cop_dem_id(product_id: str) -> List[int]:
|
|
658
682
|
parts = product_id.split("_")
|
|
@@ -670,17 +694,25 @@ def format_metadata(search_param: str, *args: Tuple[Any], **kwargs: Any) -> str:
|
|
|
670
694
|
return bbox
|
|
671
695
|
|
|
672
696
|
@staticmethod
|
|
673
|
-
def
|
|
674
|
-
|
|
675
|
-
|
|
676
|
-
|
|
697
|
+
def convert_dates_from_cmems_id(product_id: str):
|
|
698
|
+
date_format_1 = "[0-9]{10}"
|
|
699
|
+
date_format_2 = "[0-9]{8}"
|
|
700
|
+
dates = re.findall(date_format_1, product_id)
|
|
701
|
+
if dates:
|
|
702
|
+
date = dates[0]
|
|
703
|
+
else:
|
|
704
|
+
dates = re.findall(date_format_2, product_id)
|
|
705
|
+
date = dates[0]
|
|
706
|
+
if len(date) == 10:
|
|
707
|
+
date_time = datetime.strptime(dates[0], "%Y%m%d%H")
|
|
677
708
|
else:
|
|
678
|
-
|
|
679
|
-
|
|
680
|
-
|
|
681
|
-
|
|
682
|
-
|
|
683
|
-
|
|
709
|
+
date_time = datetime.strptime(dates[0], "%Y%m%d")
|
|
710
|
+
return {
|
|
711
|
+
"min_date": date_time.strftime("%Y-%m-%dT%H:%M:%SZ"),
|
|
712
|
+
"max_date": (date_time + timedelta(days=1)).strftime(
|
|
713
|
+
"%Y-%m-%dT%H:%M:%SZ"
|
|
714
|
+
),
|
|
715
|
+
}
|
|
684
716
|
|
|
685
717
|
@staticmethod
|
|
686
718
|
def convert_to_datetime_dict(
|
|
@@ -791,7 +823,10 @@ def format_metadata(search_param: str, *args: Tuple[Any], **kwargs: Any) -> str:
|
|
|
791
823
|
@staticmethod
|
|
792
824
|
def convert_get_dates_from_string(text: str, split_param="-"):
|
|
793
825
|
reg = "[0-9]{8}" + split_param + "[0-9]{8}"
|
|
794
|
-
|
|
826
|
+
match = re.search(reg, text)
|
|
827
|
+
if not match:
|
|
828
|
+
return NOT_AVAILABLE
|
|
829
|
+
dates_str = match.group()
|
|
795
830
|
dates = dates_str.split(split_param)
|
|
796
831
|
start_date = datetime.strptime(dates[0], "%Y%m%d")
|
|
797
832
|
end_date = datetime.strptime(dates[1], "%Y%m%d")
|
|
@@ -800,6 +835,79 @@ def format_metadata(search_param: str, *args: Tuple[Any], **kwargs: Any) -> str:
|
|
|
800
835
|
"endDate": end_date.strftime("%Y-%m-%dT%H:%M:%SZ"),
|
|
801
836
|
}
|
|
802
837
|
|
|
838
|
+
@staticmethod
|
|
839
|
+
def convert_get_hydrological_year(date: str):
|
|
840
|
+
utc_date = MetadataFormatter.convert_to_iso_utc_datetime(date)
|
|
841
|
+
date_object = datetime.strptime(utc_date, "%Y-%m-%dT%H:%M:%S.%fZ")
|
|
842
|
+
date_object_second_year = date_object + timedelta(days=365)
|
|
843
|
+
return [
|
|
844
|
+
f'{date_object.strftime("%Y")}_{date_object_second_year.strftime("%y")}'
|
|
845
|
+
]
|
|
846
|
+
|
|
847
|
+
@staticmethod
|
|
848
|
+
def convert_get_variables_from_path(path: str):
|
|
849
|
+
if "?" not in path:
|
|
850
|
+
return []
|
|
851
|
+
variables = path.split("?")[1]
|
|
852
|
+
return variables.split(",")
|
|
853
|
+
|
|
854
|
+
@staticmethod
|
|
855
|
+
def convert_assets_list_to_dict(
|
|
856
|
+
assets_list: List[Dict[str, str]], asset_name_key: str = "title"
|
|
857
|
+
) -> Dict[str, Dict[str, str]]:
|
|
858
|
+
"""Convert a list of assets to a dictionary where keys represent
|
|
859
|
+
name of assets and are found among values of asset dictionaries.
|
|
860
|
+
|
|
861
|
+
assets_list == [
|
|
862
|
+
{"href": "foo", "title": "asset1", "name": "foo-name"},
|
|
863
|
+
{"href": "bar", "title": "path/to/asset1", "name": "bar-name"},
|
|
864
|
+
{"href": "baz", "title": "path/to/asset2", "name": "baz-name"},
|
|
865
|
+
{"href": "qux", "title": "asset3", "name": "qux-name"},
|
|
866
|
+
] and asset_name_key == "title" => {
|
|
867
|
+
"asset1": {"href": "foo", "title": "asset1", "name": "foo-name"},
|
|
868
|
+
"path/to/asset1": {"href": "bar", "title": "path/to/asset1", "name": "bar-name"},
|
|
869
|
+
"asset2": {"href": "baz", "title": "path/to/asset2", "name": "baz-name"},
|
|
870
|
+
"asset3": {"href": "qux", "title": "asset3", "name": "qux-name"},
|
|
871
|
+
}
|
|
872
|
+
assets_list == [
|
|
873
|
+
{"href": "foo", "title": "foo-title", "name": "asset1"},
|
|
874
|
+
{"href": "bar", "title": "bar-title", "name": "path/to/asset1"},
|
|
875
|
+
{"href": "baz", "title": "baz-title", "name": "path/to/asset2"},
|
|
876
|
+
{"href": "qux", "title": "qux-title", "name": "asset3"},
|
|
877
|
+
] and asset_name_key == "name" => {
|
|
878
|
+
"asset1": {"href": "foo", "title": "foo-title", "name": "asset1"},
|
|
879
|
+
"path/to/asset1": {"href": "bar", "title": "bar-title", "name": "path/to/asset1"},
|
|
880
|
+
"asset2": {"href": "baz", "title": "baz-title", "name": "path/to/asset2"},
|
|
881
|
+
"asset3": {"href": "qux", "title": "qux-title", "name": "asset3"},
|
|
882
|
+
}
|
|
883
|
+
"""
|
|
884
|
+
asset_names: List[str] = []
|
|
885
|
+
assets_dict: Dict[str, Dict[str, str]] = {}
|
|
886
|
+
|
|
887
|
+
for asset in assets_list:
|
|
888
|
+
asset_name = asset[asset_name_key]
|
|
889
|
+
asset_names.append(asset_name)
|
|
890
|
+
assets_dict[asset_name] = asset
|
|
891
|
+
|
|
892
|
+
# we only keep the equivalent of the path basename in the case where the
|
|
893
|
+
# asset name has a path pattern and this basename is only found once
|
|
894
|
+
immutable_asset_indexes: List[int] = []
|
|
895
|
+
for i, asset_name in enumerate(asset_names):
|
|
896
|
+
if i in immutable_asset_indexes:
|
|
897
|
+
continue
|
|
898
|
+
change_asset_name = True
|
|
899
|
+
asset_basename = asset_name.split("/")[-1]
|
|
900
|
+
j = i + 1
|
|
901
|
+
while change_asset_name and j < len(asset_names):
|
|
902
|
+
asset_tmp_basename = asset_names[j].split("/")[-1]
|
|
903
|
+
if asset_basename == asset_tmp_basename:
|
|
904
|
+
change_asset_name = False
|
|
905
|
+
immutable_asset_indexes.extend([i, j])
|
|
906
|
+
j += 1
|
|
907
|
+
if change_asset_name:
|
|
908
|
+
assets_dict[asset_basename] = assets_dict.pop(asset_name)
|
|
909
|
+
return assets_dict
|
|
910
|
+
|
|
803
911
|
# if stac extension colon separator `:` is in search params, parse it to prevent issues with vformat
|
|
804
912
|
if re.search(r"{[a-zA-Z0-9_-]*:[a-zA-Z0-9_-]*}", search_param):
|
|
805
913
|
search_param = re.sub(
|
|
@@ -840,7 +948,7 @@ def properties_from_json(
|
|
|
840
948
|
else:
|
|
841
949
|
conversion_or_none, path_or_text = value
|
|
842
950
|
if isinstance(path_or_text, str):
|
|
843
|
-
if re.search(r"({[^{}]+})+", path_or_text):
|
|
951
|
+
if re.search(r"({[^{}:]+})+", path_or_text):
|
|
844
952
|
templates[metadata] = path_or_text
|
|
845
953
|
else:
|
|
846
954
|
properties[metadata] = path_or_text
|
|
@@ -874,7 +982,7 @@ def properties_from_json(
|
|
|
874
982
|
conversion_or_none = conversion_or_none[0]
|
|
875
983
|
|
|
876
984
|
# check if conversion uses variables to format
|
|
877
|
-
if re.search(r"({[^{}]+})+", conversion_or_none):
|
|
985
|
+
if re.search(r"({[^{}:]+})+", conversion_or_none):
|
|
878
986
|
conversion_or_none = conversion_or_none.format(**properties)
|
|
879
987
|
|
|
880
988
|
properties[metadata] = format_metadata(
|
|
@@ -890,7 +998,7 @@ def properties_from_json(
|
|
|
890
998
|
# Resolve templates
|
|
891
999
|
for metadata, template in templates.items():
|
|
892
1000
|
try:
|
|
893
|
-
properties[metadata] = template
|
|
1001
|
+
properties[metadata] = format_string(metadata, template, **properties)
|
|
894
1002
|
except ValueError:
|
|
895
1003
|
logger.warning(
|
|
896
1004
|
f"Could not parse {metadata} ({template}) using product properties"
|
|
@@ -905,13 +1013,18 @@ def properties_from_json(
|
|
|
905
1013
|
discovery_pattern = discovery_config.get("metadata_pattern", None)
|
|
906
1014
|
discovery_path = discovery_config.get("metadata_path", None)
|
|
907
1015
|
if discovery_pattern and discovery_path:
|
|
908
|
-
|
|
1016
|
+
discovery_jsonpath = string_to_jsonpath(discovery_path)
|
|
1017
|
+
discovered_properties = (
|
|
1018
|
+
discovery_jsonpath.find(json)
|
|
1019
|
+
if isinstance(discovery_jsonpath, JSONPath)
|
|
1020
|
+
else []
|
|
1021
|
+
)
|
|
909
1022
|
for found_jsonpath in discovered_properties:
|
|
910
1023
|
if "metadata_path_id" in discovery_config.keys():
|
|
911
1024
|
found_key_paths = string_to_jsonpath(
|
|
912
1025
|
discovery_config["metadata_path_id"], force=True
|
|
913
1026
|
).find(found_jsonpath.value)
|
|
914
|
-
if not found_key_paths:
|
|
1027
|
+
if not found_key_paths or isinstance(found_key_paths, int):
|
|
915
1028
|
continue
|
|
916
1029
|
found_key = found_key_paths[0].value
|
|
917
1030
|
used_jsonpath = Child(
|
|
@@ -934,7 +1047,9 @@ def properties_from_json(
|
|
|
934
1047
|
discovery_config["metadata_path_value"], force=True
|
|
935
1048
|
).find(found_jsonpath.value)
|
|
936
1049
|
properties[found_key] = (
|
|
937
|
-
found_value_path[0].value
|
|
1050
|
+
found_value_path[0].value
|
|
1051
|
+
if found_value_path and not isinstance(found_value_path, int)
|
|
1052
|
+
else NOT_AVAILABLE
|
|
938
1053
|
)
|
|
939
1054
|
else:
|
|
940
1055
|
# default value got from metadata_path
|
|
@@ -950,7 +1065,7 @@ def properties_from_json(
|
|
|
950
1065
|
|
|
951
1066
|
|
|
952
1067
|
def properties_from_xml(
|
|
953
|
-
xml_as_text:
|
|
1068
|
+
xml_as_text: AnyStr,
|
|
954
1069
|
mapping: Any,
|
|
955
1070
|
empty_ns_prefix: str = "ns",
|
|
956
1071
|
discovery_config: Optional[Dict[str, Any]] = None,
|
|
@@ -1051,7 +1166,7 @@ def properties_from_xml(
|
|
|
1051
1166
|
conversion_or_none = conversion_or_none[0]
|
|
1052
1167
|
|
|
1053
1168
|
# check if conversion uses variables to format
|
|
1054
|
-
if re.search(r"({[^{}]+})+", conversion_or_none):
|
|
1169
|
+
if re.search(r"({[^{}:]+})+", conversion_or_none):
|
|
1055
1170
|
conversion_or_none = conversion_or_none.format(**properties)
|
|
1056
1171
|
|
|
1057
1172
|
properties[metadata] = [
|
|
@@ -1073,7 +1188,7 @@ def properties_from_xml(
|
|
|
1073
1188
|
# formatting resolution using previously successfully resolved properties
|
|
1074
1189
|
# Ignore any transformation specified. If a value is to be passed as is,
|
|
1075
1190
|
# we don't want to transform it further
|
|
1076
|
-
if re.search(r"({[^{}]+})+", path_or_text):
|
|
1191
|
+
if re.search(r"({[^{}:]+})+", path_or_text):
|
|
1077
1192
|
templates[metadata] = path_or_text
|
|
1078
1193
|
else:
|
|
1079
1194
|
properties[metadata] = path_or_text
|
|
@@ -1146,7 +1261,7 @@ def mtd_cfg_as_conversion_and_querypath(
|
|
|
1146
1261
|
else:
|
|
1147
1262
|
parsed_path = path
|
|
1148
1263
|
|
|
1149
|
-
if len(dest_dict[metadata]) == 2:
|
|
1264
|
+
if isinstance(dest_dict[metadata], list) and len(dest_dict[metadata]) == 2:
|
|
1150
1265
|
dest_dict[metadata][1] = (conversion, parsed_path)
|
|
1151
1266
|
else:
|
|
1152
1267
|
dest_dict[metadata] = (conversion, parsed_path)
|
|
@@ -1158,13 +1273,13 @@ def mtd_cfg_as_conversion_and_querypath(
|
|
|
1158
1273
|
|
|
1159
1274
|
|
|
1160
1275
|
def format_query_params(
|
|
1161
|
-
product_type: str, config: PluginConfig,
|
|
1276
|
+
product_type: str, config: PluginConfig, query_dict: Dict[str, Any]
|
|
1162
1277
|
) -> Dict[str, Any]:
|
|
1163
1278
|
"""format the search parameters to query parameters"""
|
|
1164
|
-
if "raise_errors" in
|
|
1165
|
-
del
|
|
1279
|
+
if "raise_errors" in query_dict.keys():
|
|
1280
|
+
del query_dict["raise_errors"]
|
|
1166
1281
|
# . not allowed in eodag_search_key, replaced with %2E
|
|
1167
|
-
|
|
1282
|
+
query_dict = {k.replace(".", "%2E"): v for k, v in query_dict.items()}
|
|
1168
1283
|
|
|
1169
1284
|
product_type_metadata_mapping = dict(
|
|
1170
1285
|
config.metadata_mapping,
|
|
@@ -1174,16 +1289,16 @@ def format_query_params(
|
|
|
1174
1289
|
query_params: Dict[str, Any] = {}
|
|
1175
1290
|
# Get all the search parameters that are recognised as queryables by the
|
|
1176
1291
|
# provider (they appear in the queryables dictionary)
|
|
1177
|
-
queryables = _get_queryables(
|
|
1292
|
+
queryables = _get_queryables(query_dict, config, product_type_metadata_mapping)
|
|
1178
1293
|
|
|
1179
1294
|
for eodag_search_key, provider_search_key in queryables.items():
|
|
1180
|
-
user_input =
|
|
1295
|
+
user_input = query_dict[eodag_search_key]
|
|
1181
1296
|
|
|
1182
1297
|
if COMPLEX_QS_REGEX.match(provider_search_key):
|
|
1183
1298
|
parts = provider_search_key.split("=")
|
|
1184
1299
|
if len(parts) == 1:
|
|
1185
1300
|
formatted_query_param = format_metadata(
|
|
1186
|
-
provider_search_key, product_type, **
|
|
1301
|
+
provider_search_key, product_type, **query_dict
|
|
1187
1302
|
)
|
|
1188
1303
|
formatted_query_param = formatted_query_param.replace("'", '"')
|
|
1189
1304
|
if "{{" in provider_search_key:
|
|
@@ -1202,7 +1317,7 @@ def format_query_params(
|
|
|
1202
1317
|
else:
|
|
1203
1318
|
provider_search_key, provider_value = parts
|
|
1204
1319
|
query_params.setdefault(provider_search_key, []).append(
|
|
1205
|
-
format_metadata(provider_value, product_type, **
|
|
1320
|
+
format_metadata(provider_value, product_type, **query_dict)
|
|
1206
1321
|
)
|
|
1207
1322
|
else:
|
|
1208
1323
|
query_params[provider_search_key] = user_input
|
|
@@ -1221,7 +1336,7 @@ def format_query_params(
|
|
|
1221
1336
|
**config.products.get(product_type, {}).get("metadata_mapping", {}),
|
|
1222
1337
|
)
|
|
1223
1338
|
literal_search_params.update(
|
|
1224
|
-
_format_free_text_search(config, product_type_metadata_mapping, **
|
|
1339
|
+
_format_free_text_search(config, product_type_metadata_mapping, **query_dict)
|
|
1225
1340
|
)
|
|
1226
1341
|
for provider_search_key, provider_value in literal_search_params.items():
|
|
1227
1342
|
if isinstance(provider_value, list):
|
eodag/api/search_result.py
CHANGED
|
@@ -40,12 +40,17 @@ class SearchResult(UserList):
|
|
|
40
40
|
|
|
41
41
|
:param products: A list of products resulting from a search
|
|
42
42
|
:type products: list(:class:`~eodag.api.product._product.EOProduct`)
|
|
43
|
+
:param number_matched: (optional) the estimated total number of matching results
|
|
44
|
+
:type number_matched: Optional[int]
|
|
43
45
|
"""
|
|
44
46
|
|
|
45
47
|
data: List[EOProduct]
|
|
46
48
|
|
|
47
|
-
def __init__(
|
|
49
|
+
def __init__(
|
|
50
|
+
self, products: List[EOProduct], number_matched: Optional[int] = None
|
|
51
|
+
) -> None:
|
|
48
52
|
super(SearchResult, self).__init__(products)
|
|
53
|
+
self.number_matched = number_matched
|
|
49
54
|
|
|
50
55
|
def crunch(self, cruncher: Crunch, **search_params: Any) -> SearchResult:
|
|
51
56
|
"""Do some crunching with the underlying EO products.
|
|
@@ -168,3 +173,45 @@ class SearchResult(UserList):
|
|
|
168
173
|
See https://gist.github.com/sgillies/2217756
|
|
169
174
|
"""
|
|
170
175
|
return self.as_geojson_object()
|
|
176
|
+
|
|
177
|
+
def _repr_html_(self):
|
|
178
|
+
total_count = f"/{self.number_matched}" if self.number_matched else ""
|
|
179
|
+
return (
|
|
180
|
+
f"""<table>
|
|
181
|
+
<thead><tr><td style='text-align: left; color: grey;'>
|
|
182
|
+
{type(self).__name__} ({len(self)}{total_count})
|
|
183
|
+
</td></tr></thead>
|
|
184
|
+
"""
|
|
185
|
+
+ "".join(
|
|
186
|
+
[
|
|
187
|
+
f"""<tr><td style='text-align: left;'>
|
|
188
|
+
<details><summary style='color: grey; font-family: monospace;'>
|
|
189
|
+
{i} 
|
|
190
|
+
{type(p).__name__}(id=<span style='color: black;'>{
|
|
191
|
+
p.properties['id']
|
|
192
|
+
}</span>, provider={p.provider})
|
|
193
|
+
</summary>
|
|
194
|
+
{p._repr_html_()}
|
|
195
|
+
</details>
|
|
196
|
+
</td></tr>
|
|
197
|
+
"""
|
|
198
|
+
for i, p in enumerate(self)
|
|
199
|
+
]
|
|
200
|
+
)
|
|
201
|
+
+ "</table>"
|
|
202
|
+
)
|
|
203
|
+
|
|
204
|
+
|
|
205
|
+
class RawSearchResult(UserList):
|
|
206
|
+
"""An object representing a collection of raw/unparsed search results obtained from a provider.
|
|
207
|
+
|
|
208
|
+
:param results: A list of raw/unparsed search results
|
|
209
|
+
:type results: List[Any]
|
|
210
|
+
"""
|
|
211
|
+
|
|
212
|
+
data: List[Any]
|
|
213
|
+
query_params: Dict[str, Any]
|
|
214
|
+
product_type_def_params: Dict[str, Any]
|
|
215
|
+
|
|
216
|
+
def __init__(self, results: List[Any]) -> None:
|
|
217
|
+
super(RawSearchResult, self).__init__(results)
|
eodag/cli.py
CHANGED
|
@@ -50,7 +50,6 @@ from importlib.metadata import metadata
|
|
|
50
50
|
from typing import TYPE_CHECKING, Any, Dict, List, Mapping, Set
|
|
51
51
|
|
|
52
52
|
import click
|
|
53
|
-
import uvicorn
|
|
54
53
|
|
|
55
54
|
from eodag.api.core import EODataAccessGateway
|
|
56
55
|
from eodag.utils import DEFAULT_ITEMS_PER_PAGE, DEFAULT_PAGE, parse_qs
|
|
@@ -242,6 +241,11 @@ def version() -> None:
|
|
|
242
241
|
"or a maximum value defined internally for the requested provider, or a default "
|
|
243
242
|
"maximum value equals to 50.",
|
|
244
243
|
)
|
|
244
|
+
@click.option(
|
|
245
|
+
"--count",
|
|
246
|
+
is_flag=True,
|
|
247
|
+
help="Whether to run a query with a count request or not.",
|
|
248
|
+
)
|
|
245
249
|
@click.option(
|
|
246
250
|
"--locations",
|
|
247
251
|
type=str,
|
|
@@ -334,6 +338,8 @@ def search_crunch(ctx: Context, **kwargs: Any) -> None:
|
|
|
334
338
|
if locs_file:
|
|
335
339
|
locs_file = click.format_filename(locs_file)
|
|
336
340
|
|
|
341
|
+
count = kwargs.pop("count")
|
|
342
|
+
|
|
337
343
|
# Process inputs for crunch
|
|
338
344
|
cruncher_names: Set[Any] = set(kwargs.pop("cruncher") or [])
|
|
339
345
|
cruncher_args = kwargs.pop("cruncher_args")
|
|
@@ -361,10 +367,13 @@ def search_crunch(ctx: Context, **kwargs: Any) -> None:
|
|
|
361
367
|
items_per_page = (
|
|
362
368
|
DEFAULT_ITEMS_PER_PAGE if items_per_page is None else items_per_page
|
|
363
369
|
)
|
|
364
|
-
results
|
|
365
|
-
page=page, items_per_page=items_per_page, **criteria
|
|
370
|
+
results = gateway.search(
|
|
371
|
+
count=count, page=page, items_per_page=items_per_page, **criteria
|
|
366
372
|
)
|
|
367
|
-
|
|
373
|
+
if results.number_matched is not None:
|
|
374
|
+
click.echo(
|
|
375
|
+
"Found a total number of {} products".format(results.number_matched)
|
|
376
|
+
)
|
|
368
377
|
click.echo("Returned {} products".format(len(results)))
|
|
369
378
|
|
|
370
379
|
# Crunch !
|
|
@@ -446,8 +455,6 @@ def list_pt(ctx: Context, **kwargs: Any) -> None:
|
|
|
446
455
|
provider=provider, fetch_providers=fetch_providers
|
|
447
456
|
)
|
|
448
457
|
if pt["ID"] in guessed_product_types
|
|
449
|
-
or "alias" in pt
|
|
450
|
-
and pt["alias"] in guessed_product_types
|
|
451
458
|
]
|
|
452
459
|
else:
|
|
453
460
|
product_types = dag.list_product_types(
|
|
@@ -645,6 +652,13 @@ def serve_rest(
|
|
|
645
652
|
) -> None:
|
|
646
653
|
"""Serve EODAG functionalities through a WEB interface"""
|
|
647
654
|
setup_logging(verbose=ctx.obj["verbosity"])
|
|
655
|
+
try:
|
|
656
|
+
import uvicorn
|
|
657
|
+
except ImportError:
|
|
658
|
+
raise ImportError(
|
|
659
|
+
"Feature not available, please install eodag[server] or eodag[all]"
|
|
660
|
+
)
|
|
661
|
+
|
|
648
662
|
# Set the settings of the app
|
|
649
663
|
# IMPORTANT: the order of imports counts here (first we override the settings,
|
|
650
664
|
# then we import the app so that the updated settings is taken into account in
|