eodag 3.10.0__py3-none-any.whl → 4.0.0a1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- eodag/api/core.py +378 -419
- eodag/api/product/__init__.py +3 -3
- eodag/api/product/_product.py +68 -40
- eodag/api/product/drivers/__init__.py +3 -5
- eodag/api/product/drivers/base.py +1 -18
- eodag/api/product/metadata_mapping.py +151 -215
- eodag/api/search_result.py +13 -7
- eodag/cli.py +72 -395
- eodag/config.py +46 -50
- eodag/plugins/apis/base.py +2 -2
- eodag/plugins/apis/ecmwf.py +20 -21
- eodag/plugins/apis/usgs.py +37 -33
- eodag/plugins/authentication/base.py +1 -3
- eodag/plugins/crunch/filter_date.py +3 -3
- eodag/plugins/crunch/filter_latest_intersect.py +2 -2
- eodag/plugins/crunch/filter_latest_tpl_name.py +1 -1
- eodag/plugins/download/aws.py +45 -41
- eodag/plugins/download/base.py +13 -14
- eodag/plugins/download/http.py +65 -65
- eodag/plugins/manager.py +28 -29
- eodag/plugins/search/__init__.py +3 -4
- eodag/plugins/search/base.py +128 -77
- eodag/plugins/search/build_search_result.py +105 -107
- eodag/plugins/search/cop_marine.py +44 -47
- eodag/plugins/search/csw.py +33 -33
- eodag/plugins/search/qssearch.py +335 -354
- eodag/plugins/search/stac_list_assets.py +1 -1
- eodag/plugins/search/static_stac_search.py +31 -31
- eodag/resources/{product_types.yml → collections.yml} +2353 -2429
- eodag/resources/ext_collections.json +1 -1
- eodag/resources/providers.yml +2427 -2719
- eodag/resources/stac_provider.yml +46 -90
- eodag/types/queryables.py +55 -91
- eodag/types/search_args.py +1 -1
- eodag/utils/__init__.py +94 -21
- eodag/utils/exceptions.py +6 -6
- eodag/utils/free_text_search.py +3 -3
- {eodag-3.10.0.dist-info → eodag-4.0.0a1.dist-info}/METADATA +10 -87
- eodag-4.0.0a1.dist-info/RECORD +92 -0
- {eodag-3.10.0.dist-info → eodag-4.0.0a1.dist-info}/entry_points.txt +0 -4
- eodag/plugins/authentication/oauth.py +0 -60
- eodag/plugins/download/creodias_s3.py +0 -71
- eodag/plugins/download/s3rest.py +0 -351
- eodag/plugins/search/data_request_search.py +0 -565
- eodag/resources/stac.yml +0 -294
- eodag/resources/stac_api.yml +0 -2105
- eodag/rest/__init__.py +0 -24
- eodag/rest/cache.py +0 -70
- eodag/rest/config.py +0 -67
- eodag/rest/constants.py +0 -26
- eodag/rest/core.py +0 -764
- eodag/rest/errors.py +0 -210
- eodag/rest/server.py +0 -604
- eodag/rest/server.wsgi +0 -6
- eodag/rest/stac.py +0 -1032
- eodag/rest/templates/README +0 -1
- eodag/rest/types/__init__.py +0 -18
- eodag/rest/types/collections_search.py +0 -44
- eodag/rest/types/eodag_search.py +0 -386
- eodag/rest/types/queryables.py +0 -174
- eodag/rest/types/stac_search.py +0 -272
- eodag/rest/utils/__init__.py +0 -207
- eodag/rest/utils/cql_evaluate.py +0 -119
- eodag/rest/utils/rfc3339.py +0 -64
- eodag-3.10.0.dist-info/RECORD +0 -116
- {eodag-3.10.0.dist-info → eodag-4.0.0a1.dist-info}/WHEEL +0 -0
- {eodag-3.10.0.dist-info → eodag-4.0.0a1.dist-info}/licenses/LICENSE +0 -0
- {eodag-3.10.0.dist-info → eodag-4.0.0a1.dist-info}/top_level.txt +0 -0
|
@@ -42,7 +42,6 @@ from shapely.ops import transform
|
|
|
42
42
|
from eodag.types.queryables import Queryables
|
|
43
43
|
from eodag.utils import (
|
|
44
44
|
DEFAULT_PROJ,
|
|
45
|
-
_deprecated,
|
|
46
45
|
deepcopy,
|
|
47
46
|
dict_items_recursive_apply,
|
|
48
47
|
format_string,
|
|
@@ -58,6 +57,8 @@ from eodag.utils.dates import get_timestamp
|
|
|
58
57
|
from eodag.utils.exceptions import ValidationError
|
|
59
58
|
|
|
60
59
|
if TYPE_CHECKING:
|
|
60
|
+
from collections.abc import Mapping, Sequence
|
|
61
|
+
|
|
61
62
|
from shapely.geometry.base import BaseGeometry
|
|
62
63
|
|
|
63
64
|
from eodag.config import PluginConfig
|
|
@@ -70,9 +71,9 @@ INGEST_CONVERSION_REGEX = re.compile(
|
|
|
70
71
|
)
|
|
71
72
|
NOT_AVAILABLE = "Not Available"
|
|
72
73
|
NOT_MAPPED = "Not Mapped"
|
|
73
|
-
ONLINE_STATUS = "
|
|
74
|
-
STAGING_STATUS = "
|
|
75
|
-
OFFLINE_STATUS = "
|
|
74
|
+
ONLINE_STATUS = "succeeded"
|
|
75
|
+
STAGING_STATUS = "ordered"
|
|
76
|
+
OFFLINE_STATUS = "orderable"
|
|
76
77
|
COORDS_ROUNDING_PRECISION = 4
|
|
77
78
|
WKT_MAX_LEN = 1600
|
|
78
79
|
COMPLEX_QS_REGEX = re.compile(r"^(.+=)?([^=]*)({.+})+([^=&]*)$")
|
|
@@ -96,23 +97,23 @@ def get_metadata_path(
|
|
|
96
97
|
search:
|
|
97
98
|
...
|
|
98
99
|
metadata_mapping:
|
|
99
|
-
|
|
100
|
-
-
|
|
101
|
-
- $.properties.
|
|
100
|
+
platform:
|
|
101
|
+
- platform
|
|
102
|
+
- $.properties.platform
|
|
102
103
|
id: $.properties.id
|
|
103
104
|
...
|
|
104
105
|
...
|
|
105
106
|
...
|
|
106
107
|
|
|
107
|
-
Then the metadata `id` is not queryable for this provider meanwhile `
|
|
108
|
-
is queryable. The first value of the `metadata_mapping.
|
|
109
|
-
eodag search parameter `
|
|
108
|
+
Then the metadata `id` is not queryable for this provider meanwhile `platform`
|
|
109
|
+
is queryable. The first value of the `metadata_mapping.platform` is how the
|
|
110
|
+
eodag search parameter `platform` is interpreted in the
|
|
110
111
|
:class:`~eodag.plugins.search.base.Search` plugin implemented by `provider`, and is
|
|
111
112
|
used when eodag delegates search process to the corresponding plugin.
|
|
112
113
|
|
|
113
114
|
:param map_value: The value originating from the definition of `metadata_mapping`
|
|
114
115
|
in the provider search config. For example, it is the list
|
|
115
|
-
`['
|
|
116
|
+
`['platform', '$.properties.platform']` with the sample
|
|
116
117
|
above. Or the string `$.properties.id`.
|
|
117
118
|
:returns: Either, None and the path to the metadata value, or a list of converter
|
|
118
119
|
and its args, and the path to the metadata value.
|
|
@@ -159,15 +160,19 @@ def format_metadata(search_param: str, *args: Any, **kwargs: Any) -> str:
|
|
|
159
160
|
- ``from_georss``: convert GeoRSS to shapely geometry / WKT in DEFAULT_PROJ
|
|
160
161
|
- ``get_ecmwf_time``: get the time of a datetime string in the ECMWF format
|
|
161
162
|
- ``get_group_name``: get the matching regex group name
|
|
163
|
+
- ``not_available``: replace value with "Not Available"
|
|
162
164
|
- ``recursive_sub_str``: recursively substitue in the structure (e.g. dict) values matching a regex
|
|
163
165
|
- ``remove_extension``: on a string that contains dots, only take the first part of the list obtained by
|
|
164
166
|
splitting the string on dots
|
|
165
167
|
- ``replace_str``: execute "string".replace(old, new)
|
|
168
|
+
- ``replace_str_tuple``: apply multiple replacements on a string (parts or complete)
|
|
169
|
+
- ``replace_tuple``: apply multiple replacements matching whole value
|
|
166
170
|
- ``s2msil2a_title_to_aws_productinfo``: used to generate SAFE format metadata for data from AWS
|
|
167
171
|
- ``sanitize``: sanitize string
|
|
168
172
|
- ``slice_str``: slice a string (equivalent to s[start, end, step])
|
|
173
|
+
- ``split``: split a string using given separator
|
|
169
174
|
- ``split_cop_dem_id``: get the bbox by splitting the product id
|
|
170
|
-
- ``split_corine_id``: get the
|
|
175
|
+
- ``split_corine_id``: get the collection by splitting the product id
|
|
171
176
|
- ``to_bounds_lists``: convert to list(s) of bounds
|
|
172
177
|
- ``to_datetime_dict``: convert a datetime string to a dictionary where values are either a string or a list
|
|
173
178
|
- ``to_ewkt``: convert to EWKT (Extended Well-Known text)
|
|
@@ -199,6 +204,44 @@ def format_metadata(search_param: str, *args: Any, **kwargs: Any) -> str:
|
|
|
199
204
|
self.custom_converter: Optional[Callable] = None
|
|
200
205
|
self.custom_args: Optional[str] = None
|
|
201
206
|
|
|
207
|
+
def parse(self, format_string: str):
|
|
208
|
+
"""
|
|
209
|
+
Rewrite field names in the template before the base parser sees them.
|
|
210
|
+
Replaces `{foo:bar}` with `{foo__bar}`.
|
|
211
|
+
"""
|
|
212
|
+
pattern = re.compile(r"{([^{}]+)}")
|
|
213
|
+
|
|
214
|
+
def rewrite_field(field: str) -> str:
|
|
215
|
+
# If there's a format spec (e.g., {foo:bar:.2f}), preserve it
|
|
216
|
+
if ":" in field and not field.lstrip().startswith(("!", ".", ":")):
|
|
217
|
+
before_colon, *after = field.split(":")
|
|
218
|
+
# Don't confuse format spec with field name colons
|
|
219
|
+
if len(after) == 1 and "." in after[0]:
|
|
220
|
+
# It's a format specifier, leave it
|
|
221
|
+
return field
|
|
222
|
+
return field.replace(":", "__", 1)
|
|
223
|
+
return field
|
|
224
|
+
|
|
225
|
+
# Replace in string (but not in format_spec itself)
|
|
226
|
+
safe_template = pattern.sub(
|
|
227
|
+
lambda m: "{" + rewrite_field(m.group(1)) + "}", format_string
|
|
228
|
+
)
|
|
229
|
+
|
|
230
|
+
# Yield from base class
|
|
231
|
+
yield from super().parse(safe_template)
|
|
232
|
+
|
|
233
|
+
def get_value(
|
|
234
|
+
self, key: Any, args: "Sequence[Any]", kwargs: "Mapping[str, Any]"
|
|
235
|
+
) -> Any:
|
|
236
|
+
"""
|
|
237
|
+
Look up rewritten field name in kwargs by converting __ back to :
|
|
238
|
+
"""
|
|
239
|
+
if isinstance(key, str):
|
|
240
|
+
original_key = key.replace("__", ":")
|
|
241
|
+
key_with_COLON = key.replace("__", "_COLON_")
|
|
242
|
+
return kwargs.get(original_key) or kwargs.get(key_with_COLON)
|
|
243
|
+
return super().get_value(key, args, kwargs)
|
|
244
|
+
|
|
202
245
|
def get_field(self, field_name: str, args: Any, kwargs: Any) -> Any:
|
|
203
246
|
conversion_func_spec = self.CONVERSION_REGEX.match(field_name)
|
|
204
247
|
# Register a custom converter if any for later use (see convert_field)
|
|
@@ -529,11 +572,15 @@ def format_metadata(search_param: str, *args: Any, **kwargs: Any) -> str:
|
|
|
529
572
|
return re.sub(old, new, value)
|
|
530
573
|
|
|
531
574
|
@staticmethod
|
|
532
|
-
def convert_replace_str_tuple(
|
|
575
|
+
def convert_replace_str_tuple(
|
|
576
|
+
value: Union[str, dict[Any, Any]], args: str
|
|
577
|
+
) -> str:
|
|
533
578
|
"""
|
|
534
|
-
Apply multiple replacements on a string.
|
|
535
|
-
|
|
536
|
-
|
|
579
|
+
Apply multiple replacements on a string (parts or complete).
|
|
580
|
+
|
|
581
|
+
:param value: input string or dict.
|
|
582
|
+
:param args: string representing a list/tuple of (old, new) pairs, like
|
|
583
|
+
``'(("old1", "new1"), ("old2", "new2"))'``
|
|
537
584
|
"""
|
|
538
585
|
if isinstance(value, dict):
|
|
539
586
|
value = MetadataFormatter.convert_to_geojson(value)
|
|
@@ -557,6 +604,57 @@ def format_metadata(search_param: str, *args: Any, **kwargs: Any) -> str:
|
|
|
557
604
|
|
|
558
605
|
return value
|
|
559
606
|
|
|
607
|
+
@staticmethod
|
|
608
|
+
def convert_replace_tuple(value: Any, args: str) -> Any:
|
|
609
|
+
"""
|
|
610
|
+
Apply multiple replacements matching whole value.
|
|
611
|
+
|
|
612
|
+
:param value: input to replace
|
|
613
|
+
:param args: string representing a list/tuple of (old, new) pairs, like
|
|
614
|
+
``'((["old1"], "new1"), ("old2", ["new2"]))'``
|
|
615
|
+
"""
|
|
616
|
+
# args sera une chaîne représentant une liste/tuple de tuples
|
|
617
|
+
replacements = ast.literal_eval(args)
|
|
618
|
+
|
|
619
|
+
if not isinstance(replacements, (list, tuple)):
|
|
620
|
+
raise TypeError(
|
|
621
|
+
f"convert_replace_str_tuple expects a list/tuple of (old,new) pairs. "
|
|
622
|
+
f"Got {type(replacements)}: {replacements}"
|
|
623
|
+
)
|
|
624
|
+
|
|
625
|
+
for old, new in replacements:
|
|
626
|
+
if old == value:
|
|
627
|
+
return new
|
|
628
|
+
|
|
629
|
+
return value
|
|
630
|
+
|
|
631
|
+
@staticmethod
|
|
632
|
+
def convert_not_available(value: Any) -> str:
|
|
633
|
+
"""Convert any value to "Not Available".
|
|
634
|
+
|
|
635
|
+
This is more useful than "$.null" to keep original jsonpath while parsing in metadata_mapping.
|
|
636
|
+
"""
|
|
637
|
+
return NOT_AVAILABLE
|
|
638
|
+
|
|
639
|
+
@staticmethod
|
|
640
|
+
def convert_split(value: str, separator: str) -> list[str]:
|
|
641
|
+
"""Split a string using given separator"""
|
|
642
|
+
if value == NOT_AVAILABLE:
|
|
643
|
+
return [NOT_AVAILABLE]
|
|
644
|
+
if not isinstance(value, str):
|
|
645
|
+
logger.warning(
|
|
646
|
+
"Could not split non-string value %s (type %s)", value, type(value)
|
|
647
|
+
)
|
|
648
|
+
return [NOT_AVAILABLE]
|
|
649
|
+
if not isinstance(separator, str):
|
|
650
|
+
logger.warning(
|
|
651
|
+
"Could not split string using non-string separator %s (type %s)",
|
|
652
|
+
separator,
|
|
653
|
+
type(separator),
|
|
654
|
+
)
|
|
655
|
+
return [NOT_AVAILABLE]
|
|
656
|
+
return value.split(separator)
|
|
657
|
+
|
|
560
658
|
@staticmethod
|
|
561
659
|
def convert_ceda_collection_name(value: str) -> str:
|
|
562
660
|
data_regex = re.compile(r"/data/(?P<name>.+?)/?$")
|
|
@@ -653,7 +751,7 @@ def format_metadata(search_param: str, *args: Any, **kwargs: Any) -> str:
|
|
|
653
751
|
int(x.strip()) if x.strip().lstrip("-").isdigit() else None
|
|
654
752
|
for x in args.split(",")
|
|
655
753
|
]
|
|
656
|
-
return string[cmin:cmax:cstep]
|
|
754
|
+
return string[cmin:cmax:cstep] or NOT_AVAILABLE
|
|
657
755
|
|
|
658
756
|
@staticmethod
|
|
659
757
|
def convert_to_lower(string: str) -> str:
|
|
@@ -702,7 +800,7 @@ def format_metadata(search_param: str, *args: Any, **kwargs: Any) -> str:
|
|
|
702
800
|
if id_match:
|
|
703
801
|
id_dict = id_match.groupdict()
|
|
704
802
|
return (
|
|
705
|
-
"https://roda.sentinel-hub.com/sentinel-s2-l2a/tiles/%s/%s/%s/%s/%s/%s/0/{
|
|
803
|
+
"https://roda.sentinel-hub.com/sentinel-s2-l2a/tiles/%s/%s/%s/%s/%s/%s/0/{_collection}.json"
|
|
706
804
|
% (
|
|
707
805
|
id_dict["tile1"],
|
|
708
806
|
id_dict["tile2"],
|
|
@@ -716,49 +814,10 @@ def format_metadata(search_param: str, *args: Any, **kwargs: Any) -> str:
|
|
|
716
814
|
logger.error("Could not extract title infos from %s" % string)
|
|
717
815
|
return NOT_AVAILABLE
|
|
718
816
|
|
|
719
|
-
@staticmethod
|
|
720
|
-
@_deprecated(
|
|
721
|
-
reason="Method that was used in previous wekeo provider configuration, but not used anymore",
|
|
722
|
-
version="3.7.1",
|
|
723
|
-
)
|
|
724
|
-
def convert_split_id_into_s1_params(product_id: str) -> dict[str, str]:
|
|
725
|
-
parts: list[str] = re.split(r"_(?!_)", product_id)
|
|
726
|
-
if len(parts) < 9:
|
|
727
|
-
logger.error(
|
|
728
|
-
"id %s does not match expected Sentinel-1 id format", product_id
|
|
729
|
-
)
|
|
730
|
-
raise ValueError
|
|
731
|
-
params = {"sensorMode": parts[1]}
|
|
732
|
-
level = "LEVEL" + parts[3][0]
|
|
733
|
-
params["processingLevel"] = level
|
|
734
|
-
start_date = datetime.strptime(parts[4], "%Y%m%dT%H%M%S") - timedelta(
|
|
735
|
-
seconds=1
|
|
736
|
-
)
|
|
737
|
-
params["startDate"] = start_date.strftime("%Y-%m-%dT%H:%M:%SZ")
|
|
738
|
-
end_date = datetime.strptime(parts[5], "%Y%m%dT%H%M%S") + timedelta(
|
|
739
|
-
seconds=1
|
|
740
|
-
)
|
|
741
|
-
params["endDate"] = end_date.strftime("%Y-%m-%dT%H:%M:%SZ")
|
|
742
|
-
product_type = parts[2][:3]
|
|
743
|
-
if product_type == "GRD" and parts[-1] == "COG":
|
|
744
|
-
product_type = "GRD-COG"
|
|
745
|
-
elif product_type == "GRD" and parts[-2] == "CARD" and parts[-1] == "BS":
|
|
746
|
-
product_type = "CARD-BS"
|
|
747
|
-
params["productType"] = product_type
|
|
748
|
-
polarisation_mapping = {
|
|
749
|
-
"SV": "VV",
|
|
750
|
-
"SH": "HH",
|
|
751
|
-
"DH": "HH+HV",
|
|
752
|
-
"DV": "VV+VH",
|
|
753
|
-
}
|
|
754
|
-
polarisation = polarisation_mapping[parts[3][2:]]
|
|
755
|
-
params["polarisation"] = polarisation
|
|
756
|
-
return params
|
|
757
|
-
|
|
758
817
|
@staticmethod
|
|
759
818
|
def convert_split_id_into_s3_params(product_id: str) -> dict[str, str]:
|
|
760
819
|
parts: list[str] = re.split(r"_(?!_)", product_id)
|
|
761
|
-
params = {"
|
|
820
|
+
params = {"collection": product_id[4:15]}
|
|
762
821
|
dates = re.findall("[0-9]{8}T[0-9]{6}", product_id)
|
|
763
822
|
start_date = datetime.strptime(dates[0], "%Y%m%dT%H%M%S") - timedelta(
|
|
764
823
|
seconds=1
|
|
@@ -772,48 +831,6 @@ def format_metadata(search_param: str, *args: Any, **kwargs: Any) -> str:
|
|
|
772
831
|
params["sat"] = "Sentinel-" + parts[0][1:]
|
|
773
832
|
return params
|
|
774
833
|
|
|
775
|
-
@staticmethod
|
|
776
|
-
@_deprecated(
|
|
777
|
-
reason="Method that was used in previous wekeo provider configuration, but not used anymore",
|
|
778
|
-
version="3.7.1",
|
|
779
|
-
)
|
|
780
|
-
def convert_split_id_into_s5p_params(product_id: str) -> dict[str, str]:
|
|
781
|
-
parts: list[str] = re.split(r"_(?!_)", product_id)
|
|
782
|
-
params = {
|
|
783
|
-
"productType": product_id[9:19],
|
|
784
|
-
"processingMode": parts[1],
|
|
785
|
-
"processingLevel": parts[2].replace("_", ""),
|
|
786
|
-
}
|
|
787
|
-
start_date = datetime.strptime(parts[-6], "%Y%m%dT%H%M%S") - timedelta(
|
|
788
|
-
seconds=10
|
|
789
|
-
)
|
|
790
|
-
params["startDate"] = start_date.strftime("%Y-%m-%dT%H:%M:%SZ")
|
|
791
|
-
end_date = datetime.strptime(parts[-5], "%Y%m%dT%H%M%S") + timedelta(
|
|
792
|
-
seconds=10
|
|
793
|
-
)
|
|
794
|
-
params["endDate"] = end_date.strftime("%Y-%m-%dT%H:%M:%SZ")
|
|
795
|
-
return params
|
|
796
|
-
|
|
797
|
-
@staticmethod
|
|
798
|
-
@_deprecated(
|
|
799
|
-
reason="Method that was used in previous wekeo provider configuration, but not used anymore",
|
|
800
|
-
version="3.7.1",
|
|
801
|
-
)
|
|
802
|
-
def convert_split_cop_dem_id(product_id: str) -> list[int]:
|
|
803
|
-
parts = product_id.split("_")
|
|
804
|
-
lattitude = parts[3]
|
|
805
|
-
longitude = parts[5]
|
|
806
|
-
if lattitude[0] == "N":
|
|
807
|
-
lat_num = int(lattitude[1:])
|
|
808
|
-
else:
|
|
809
|
-
lat_num = -1 * int(lattitude[1:])
|
|
810
|
-
if longitude[0] == "E":
|
|
811
|
-
long_num = int(longitude[1:])
|
|
812
|
-
else:
|
|
813
|
-
long_num = -1 * int(longitude[1:])
|
|
814
|
-
bbox = [long_num - 1, lat_num - 1, long_num + 1, lat_num + 1]
|
|
815
|
-
return bbox
|
|
816
|
-
|
|
817
834
|
@staticmethod
|
|
818
835
|
def convert_dates_from_cmems_id(product_id: str):
|
|
819
836
|
date_format_1 = "[0-9]{10}"
|
|
@@ -1035,8 +1052,12 @@ def format_metadata(search_param: str, *args: Any, **kwargs: Any) -> str:
|
|
|
1035
1052
|
return assets_dict
|
|
1036
1053
|
|
|
1037
1054
|
# if stac extension colon separator `:` is in search params, parse it to prevent issues with vformat
|
|
1038
|
-
if re.search(r"{[\w-]*:[\w#-]
|
|
1039
|
-
search_param = re.sub(
|
|
1055
|
+
if re.search(r"{[\w-]*:[\w#-]*\(?.*}", search_param):
|
|
1056
|
+
search_param = re.sub(
|
|
1057
|
+
r"{([\w-]*):([\w#-]*\(?.*)}",
|
|
1058
|
+
r"{\1_COLON_\2}",
|
|
1059
|
+
search_param,
|
|
1060
|
+
)
|
|
1040
1061
|
kwargs = {k.replace(":", "_COLON_"): v for k, v in kwargs.items()}
|
|
1041
1062
|
|
|
1042
1063
|
return MetadataFormatter().vformat(search_param, args, kwargs)
|
|
@@ -1059,6 +1080,7 @@ def properties_from_json(
|
|
|
1059
1080
|
`discovery_path` (String representation of jsonpath)
|
|
1060
1081
|
:returns: The metadata of the :class:`~eodag.api.product._product.EOProduct`
|
|
1061
1082
|
"""
|
|
1083
|
+
extracted_value: Any
|
|
1062
1084
|
properties: dict[str, Any] = {}
|
|
1063
1085
|
templates = {}
|
|
1064
1086
|
used_jsonpaths = []
|
|
@@ -1069,7 +1091,7 @@ def properties_from_json(
|
|
|
1069
1091
|
else:
|
|
1070
1092
|
conversion_or_none, path_or_text = value
|
|
1071
1093
|
if isinstance(path_or_text, str):
|
|
1072
|
-
if re.search(r"
|
|
1094
|
+
if re.search(r"{[^{}]+}", path_or_text):
|
|
1073
1095
|
templates[metadata] = path_or_text
|
|
1074
1096
|
else:
|
|
1075
1097
|
properties[metadata] = path_or_text
|
|
@@ -1078,11 +1100,13 @@ def properties_from_json(
|
|
|
1078
1100
|
match = path_or_text.find(json)
|
|
1079
1101
|
except KeyError:
|
|
1080
1102
|
match = []
|
|
1081
|
-
if len(match) ==
|
|
1103
|
+
if len(match) == 0:
|
|
1104
|
+
extracted_value = NOT_AVAILABLE
|
|
1105
|
+
elif len(match) == 1:
|
|
1082
1106
|
extracted_value = match[0].value
|
|
1083
1107
|
used_jsonpaths.append(match[0].full_path)
|
|
1084
1108
|
else:
|
|
1085
|
-
extracted_value =
|
|
1109
|
+
extracted_value = [m.value for m in match]
|
|
1086
1110
|
if extracted_value is None:
|
|
1087
1111
|
properties[metadata] = None
|
|
1088
1112
|
else:
|
|
@@ -1154,6 +1178,7 @@ def properties_from_json(
|
|
|
1154
1178
|
if isinstance(discovery_jsonpath, JSONPath)
|
|
1155
1179
|
else []
|
|
1156
1180
|
)
|
|
1181
|
+
mtd_prefix = discovery_config.get("metadata_prefix", "provider")
|
|
1157
1182
|
for found_jsonpath in discovered_properties:
|
|
1158
1183
|
if "metadata_path_id" in discovery_config.keys():
|
|
1159
1184
|
found_key_paths = string_to_jsonpath(
|
|
@@ -1175,8 +1200,13 @@ def properties_from_json(
|
|
|
1175
1200
|
if (
|
|
1176
1201
|
re.compile(discovery_pattern).match(found_key)
|
|
1177
1202
|
and found_key not in properties.keys()
|
|
1203
|
+
and f"{mtd_prefix}:{found_key}" not in properties.keys()
|
|
1178
1204
|
and used_jsonpath not in used_jsonpaths
|
|
1179
1205
|
):
|
|
1206
|
+
# prepend with default STAC prefix if none is already used
|
|
1207
|
+
if ":" not in found_key:
|
|
1208
|
+
found_key = f"{mtd_prefix}:{found_key}"
|
|
1209
|
+
|
|
1180
1210
|
if "metadata_path_value" in discovery_config.keys():
|
|
1181
1211
|
found_value_path = string_to_jsonpath(
|
|
1182
1212
|
discovery_config["metadata_path_value"], force=True
|
|
@@ -1401,7 +1431,7 @@ def mtd_cfg_as_conversion_and_querypath(
|
|
|
1401
1431
|
|
|
1402
1432
|
|
|
1403
1433
|
def format_query_params(
|
|
1404
|
-
|
|
1434
|
+
collection: str,
|
|
1405
1435
|
config: PluginConfig,
|
|
1406
1436
|
query_dict: dict[str, Any],
|
|
1407
1437
|
error_context: str = "",
|
|
@@ -1412,14 +1442,14 @@ def format_query_params(
|
|
|
1412
1442
|
# . not allowed in eodag_search_key, replaced with %2E
|
|
1413
1443
|
query_dict = {k.replace(".", "%2E"): v for k, v in query_dict.items()}
|
|
1414
1444
|
|
|
1415
|
-
|
|
1445
|
+
collection_metadata_mapping = dict(
|
|
1416
1446
|
config.metadata_mapping,
|
|
1417
|
-
**config.products.get(
|
|
1447
|
+
**config.products.get(collection, {}).get("metadata_mapping", {}),
|
|
1418
1448
|
)
|
|
1419
1449
|
|
|
1420
1450
|
# Raise error if non-queryables parameters are used and raise_mtd_discovery_error configured
|
|
1421
1451
|
if (
|
|
1422
|
-
raise_mtd_discovery_error := config.products.get(
|
|
1452
|
+
raise_mtd_discovery_error := config.products.get(collection, {})
|
|
1423
1453
|
.get("discover_metadata", {})
|
|
1424
1454
|
.get("raise_mtd_discovery_error")
|
|
1425
1455
|
) is None:
|
|
@@ -1433,7 +1463,7 @@ def format_query_params(
|
|
|
1433
1463
|
queryables = _get_queryables(
|
|
1434
1464
|
query_dict,
|
|
1435
1465
|
config,
|
|
1436
|
-
|
|
1466
|
+
collection_metadata_mapping,
|
|
1437
1467
|
raise_mtd_discovery_error,
|
|
1438
1468
|
error_context,
|
|
1439
1469
|
)
|
|
@@ -1458,7 +1488,7 @@ def format_query_params(
|
|
|
1458
1488
|
parts = provider_search_param.split("=")
|
|
1459
1489
|
if len(parts) == 1:
|
|
1460
1490
|
formatted_query_param = format_metadata(
|
|
1461
|
-
provider_search_param,
|
|
1491
|
+
provider_search_param, collection, **query_dict
|
|
1462
1492
|
)
|
|
1463
1493
|
formatted_query_param = formatted_query_param.replace("'", '"')
|
|
1464
1494
|
if "{{" in provider_search_param:
|
|
@@ -1469,6 +1499,11 @@ def format_query_params(
|
|
|
1469
1499
|
formatted_query_param = remove_str_array_quotes(
|
|
1470
1500
|
formatted_query_param
|
|
1471
1501
|
)
|
|
1502
|
+
if NOT_AVAILABLE in formatted_query_param:
|
|
1503
|
+
raise ValidationError(
|
|
1504
|
+
"Could not parse %s query parameter, got %s"
|
|
1505
|
+
% (eodag_search_key, formatted_query_param)
|
|
1506
|
+
)
|
|
1472
1507
|
|
|
1473
1508
|
# json query string (for POST request)
|
|
1474
1509
|
update_nested_dict(
|
|
@@ -1482,7 +1517,7 @@ def format_query_params(
|
|
|
1482
1517
|
else:
|
|
1483
1518
|
provider_search_key, provider_value = parts
|
|
1484
1519
|
query_params[provider_search_key] = format_metadata(
|
|
1485
|
-
provider_value,
|
|
1520
|
+
provider_value, collection, **query_dict
|
|
1486
1521
|
)
|
|
1487
1522
|
else:
|
|
1488
1523
|
query_params[provider_search_param] = user_input
|
|
@@ -1496,12 +1531,12 @@ def format_query_params(
|
|
|
1496
1531
|
# Now add formatted free text search parameters (this is for cases where a
|
|
1497
1532
|
# complex query through a free text search parameter is available for the
|
|
1498
1533
|
# provider and needed for the consumer)
|
|
1499
|
-
|
|
1534
|
+
collection_metadata_mapping = dict(
|
|
1500
1535
|
config.metadata_mapping,
|
|
1501
|
-
**config.products.get(
|
|
1536
|
+
**config.products.get(collection, {}).get("metadata_mapping", {}),
|
|
1502
1537
|
)
|
|
1503
1538
|
literal_search_params.update(
|
|
1504
|
-
_format_free_text_search(config,
|
|
1539
|
+
_format_free_text_search(config, collection_metadata_mapping, **query_dict)
|
|
1505
1540
|
)
|
|
1506
1541
|
for provider_search_key, provider_value in literal_search_params.items():
|
|
1507
1542
|
if isinstance(provider_value, list):
|
|
@@ -1598,7 +1633,7 @@ def _get_queryables(
|
|
|
1598
1633
|
# raise an error when a query param not allowed by the provider is found
|
|
1599
1634
|
if not isinstance(md_mapping, list) and raise_mtd_discovery_error:
|
|
1600
1635
|
raise ValidationError(
|
|
1601
|
-
"Search parameters which are not queryable are disallowed for this
|
|
1636
|
+
"Search parameters which are not queryable are disallowed for this collection on this provider: "
|
|
1602
1637
|
f"please remove '{eodag_search_key}' from your search parameters. {error_context}",
|
|
1603
1638
|
{eodag_search_key},
|
|
1604
1639
|
)
|
|
@@ -1720,102 +1755,3 @@ def get_provider_queryable_key(
|
|
|
1720
1755
|
return ""
|
|
1721
1756
|
else:
|
|
1722
1757
|
return eodag_key
|
|
1723
|
-
|
|
1724
|
-
|
|
1725
|
-
# Keys taken from OpenSearch extension for Earth Observation http://docs.opengeospatial.org/is/13-026r9/13-026r9.html
|
|
1726
|
-
# For a metadata to be queryable, The way to query it must be specified in the
|
|
1727
|
-
# provider metadata_mapping configuration parameter. It will be automatically
|
|
1728
|
-
# detected as queryable by eodag when this is done
|
|
1729
|
-
OSEO_METADATA_MAPPING = {
|
|
1730
|
-
# Opensearch resource identifier within the search engine context (in our case
|
|
1731
|
-
# within the context of the data provider)
|
|
1732
|
-
"uid": "$.uid",
|
|
1733
|
-
# OpenSearch Parameters for Collection Search (Table 3)
|
|
1734
|
-
"productType": "$.properties.productType",
|
|
1735
|
-
"doi": "$.properties.doi",
|
|
1736
|
-
"platform": "$.properties.platform",
|
|
1737
|
-
"platformSerialIdentifier": "$.properties.platformSerialIdentifier",
|
|
1738
|
-
"instrument": "$.properties.instrument",
|
|
1739
|
-
"sensorType": "$.properties.sensorType",
|
|
1740
|
-
"compositeType": "$.properties.compositeType",
|
|
1741
|
-
"processingLevel": "$.properties.processingLevel",
|
|
1742
|
-
"orbitType": "$.properties.orbitType",
|
|
1743
|
-
"spectralRange": "$.properties.spectralRange",
|
|
1744
|
-
"wavelengths": "$.properties.wavelengths",
|
|
1745
|
-
"hasSecurityConstraints": "$.properties.hasSecurityConstraints",
|
|
1746
|
-
"dissemination": "$.properties.dissemination",
|
|
1747
|
-
# INSPIRE obligated OpenSearch Parameters for Collection Search (Table 4)
|
|
1748
|
-
"title": "$.properties.title",
|
|
1749
|
-
"topicCategory": "$.properties.topicCategory",
|
|
1750
|
-
"keyword": "$.properties.keyword",
|
|
1751
|
-
"abstract": "$.properties.abstract",
|
|
1752
|
-
"resolution": "$.properties.resolution",
|
|
1753
|
-
"organisationName": "$.properties.organisationName",
|
|
1754
|
-
"organisationRole": "$.properties.organisationRole",
|
|
1755
|
-
"publicationDate": "$.properties.publicationDate",
|
|
1756
|
-
"lineage": "$.properties.lineage",
|
|
1757
|
-
"useLimitation": "$.properties.useLimitation",
|
|
1758
|
-
"accessConstraint": "$.properties.accessConstraint",
|
|
1759
|
-
"otherConstraint": "$.properties.otherConstraint",
|
|
1760
|
-
"classification": "$.properties.classification",
|
|
1761
|
-
"language": "$.properties.language",
|
|
1762
|
-
"specification": "$.properties.specification",
|
|
1763
|
-
# OpenSearch Parameters for Product Search (Table 5)
|
|
1764
|
-
"parentIdentifier": "$.properties.parentIdentifier",
|
|
1765
|
-
"productionStatus": "$.properties.productionStatus",
|
|
1766
|
-
"acquisitionType": "$.properties.acquisitionType",
|
|
1767
|
-
"orbitNumber": "$.properties.orbitNumber",
|
|
1768
|
-
"orbitDirection": "$.properties.orbitDirection",
|
|
1769
|
-
"track": "$.properties.track",
|
|
1770
|
-
"frame": "$.properties.frame",
|
|
1771
|
-
"swathIdentifier": "$.properties.swathIdentifier",
|
|
1772
|
-
"cloudCover": "$.properties.cloudCover",
|
|
1773
|
-
"snowCover": "$.properties.snowCover",
|
|
1774
|
-
"lowestLocation": "$.properties.lowestLocation",
|
|
1775
|
-
"highestLocation": "$.properties.highestLocation",
|
|
1776
|
-
"productVersion": "$.properties.productVersion",
|
|
1777
|
-
"productQualityStatus": "$.properties.productQualityStatus",
|
|
1778
|
-
"productQualityDegradationTag": "$.properties.productQualityDegradationTag",
|
|
1779
|
-
"processorName": "$.properties.processorName",
|
|
1780
|
-
"processingCenter": "$.properties.processingCenter",
|
|
1781
|
-
"creationDate": "$.properties.creationDate",
|
|
1782
|
-
"modificationDate": "$.properties.modificationDate",
|
|
1783
|
-
"processingDate": "$.properties.processingDate",
|
|
1784
|
-
"sensorMode": "$.properties.sensorMode",
|
|
1785
|
-
"archivingCenter": "$.properties.archivingCenter",
|
|
1786
|
-
"processingMode": "$.properties.processingMode",
|
|
1787
|
-
# OpenSearch Parameters for Acquistion Parameters Search (Table 6)
|
|
1788
|
-
"availabilityTime": "$.properties.availabilityTime",
|
|
1789
|
-
"acquisitionStation": "$.properties.acquisitionStation",
|
|
1790
|
-
"acquisitionSubType": "$.properties.acquisitionSubType",
|
|
1791
|
-
"startTimeFromAscendingNode": "$.properties.startTimeFromAscendingNode",
|
|
1792
|
-
"completionTimeFromAscendingNode": "$.properties.completionTimeFromAscendingNode",
|
|
1793
|
-
"illuminationAzimuthAngle": "$.properties.illuminationAzimuthAngle",
|
|
1794
|
-
"illuminationZenithAngle": "$.properties.illuminationZenithAngle",
|
|
1795
|
-
"illuminationElevationAngle": "$.properties.illuminationElevationAngle",
|
|
1796
|
-
"polarizationMode": "$.properties.polarizationMode",
|
|
1797
|
-
"polarizationChannels": "$.properties.polarizationChannels",
|
|
1798
|
-
"antennaLookDirection": "$.properties.antennaLookDirection",
|
|
1799
|
-
"minimumIncidenceAngle": "$.properties.minimumIncidenceAngle",
|
|
1800
|
-
"maximumIncidenceAngle": "$.properties.maximumIncidenceAngle",
|
|
1801
|
-
"dopplerFrequency": "$.properties.dopplerFrequency",
|
|
1802
|
-
"incidenceAngleVariation": "$.properties.incidenceAngleVariation",
|
|
1803
|
-
}
|
|
1804
|
-
DEFAULT_METADATA_MAPPING = dict(
|
|
1805
|
-
OSEO_METADATA_MAPPING,
|
|
1806
|
-
**{
|
|
1807
|
-
# Custom parameters (not defined in the base document referenced above)
|
|
1808
|
-
# id differs from uid. The id is an identifier by which a product which is
|
|
1809
|
-
# distributed by many providers can be retrieved (a property that it has in common
|
|
1810
|
-
# in the catalogues of all the providers on which it is referenced)
|
|
1811
|
-
"id": "$.id",
|
|
1812
|
-
# The geographic extent of the product
|
|
1813
|
-
"geometry": "$.geometry",
|
|
1814
|
-
# The url of the quicklook
|
|
1815
|
-
"quicklook": "$.properties.quicklook",
|
|
1816
|
-
# The url to download the product "as is" (literal or as a template to be completed
|
|
1817
|
-
# either after the search result is obtained from the provider or during the eodag
|
|
1818
|
-
# download phase)
|
|
1819
|
-
"downloadLink": "$.properties.downloadLink",
|
|
1820
|
-
},
|
|
1821
|
-
)
|
eodag/api/search_result.py
CHANGED
|
@@ -140,7 +140,7 @@ class SearchResult(UserList[EOProduct]):
|
|
|
140
140
|
Use cruncher :class:`~eodag.plugins.crunch.filter_property.FilterProperty`,
|
|
141
141
|
filter for online products.
|
|
142
142
|
"""
|
|
143
|
-
return self.filter_property(
|
|
143
|
+
return self.filter_property(**{"order:status": "succeeded"})
|
|
144
144
|
|
|
145
145
|
@staticmethod
|
|
146
146
|
def from_geojson(feature_collection: dict[str, Any]) -> SearchResult:
|
|
@@ -260,7 +260,7 @@ def _import_stac_item_from_eodag_server(
|
|
|
260
260
|
assets = {
|
|
261
261
|
k: v["alternate"]["origin"]
|
|
262
262
|
for k, v in feature.get("assets", {}).items()
|
|
263
|
-
if k not in ("thumbnail", "downloadLink")
|
|
263
|
+
if k not in ("thumbnail", "downloadLink", "eodag:download_link")
|
|
264
264
|
and "origin" in v.get("alternate", {})
|
|
265
265
|
}
|
|
266
266
|
if assets:
|
|
@@ -274,6 +274,12 @@ def _import_stac_item_from_eodag_server(
|
|
|
274
274
|
.get("alternate", {})
|
|
275
275
|
.get("origin", {})
|
|
276
276
|
.get("href")
|
|
277
|
+
) or (
|
|
278
|
+
feature.get("assets", {})
|
|
279
|
+
.get("eodag:download_link", {})
|
|
280
|
+
.get("alternate", {})
|
|
281
|
+
.get("origin", {})
|
|
282
|
+
.get("href")
|
|
277
283
|
)
|
|
278
284
|
if download_link:
|
|
279
285
|
updated_item["assets"] = {}
|
|
@@ -328,12 +334,12 @@ def _import_stac_item_from_known_provider(
|
|
|
328
334
|
configured_pts = [
|
|
329
335
|
k
|
|
330
336
|
for k, v in search_plugin.config.products.items()
|
|
331
|
-
if v.get("
|
|
337
|
+
if v.get("_collection") == feature.get("collection")
|
|
332
338
|
]
|
|
333
339
|
if len(configured_pts) > 0:
|
|
334
|
-
eo_product.
|
|
340
|
+
eo_product.collection = configured_pts[0]
|
|
335
341
|
else:
|
|
336
|
-
eo_product.
|
|
342
|
+
eo_product.collection = feature.get("collection")
|
|
337
343
|
|
|
338
344
|
eo_product._register_downloader_from_manager(plugins_manager)
|
|
339
345
|
imported_products.append(eo_product)
|
|
@@ -359,7 +365,7 @@ def _import_stac_item_from_unknown_provider(
|
|
|
359
365
|
except MisconfiguredError:
|
|
360
366
|
pass
|
|
361
367
|
if eo_product is not None:
|
|
362
|
-
eo_product.
|
|
368
|
+
eo_product.collection = feature.get("collection")
|
|
363
369
|
eo_product._register_downloader_from_manager(plugins_manager)
|
|
364
370
|
return SearchResult([eo_product])
|
|
365
371
|
else:
|
|
@@ -373,7 +379,7 @@ class RawSearchResult(UserList[dict[str, Any]]):
|
|
|
373
379
|
"""
|
|
374
380
|
|
|
375
381
|
query_params: dict[str, Any]
|
|
376
|
-
|
|
382
|
+
collection_def_params: dict[str, Any]
|
|
377
383
|
|
|
378
384
|
def __init__(self, results: list[Any]) -> None:
|
|
379
385
|
super(RawSearchResult, self).__init__(results)
|