eodag 3.0.0b3__py3-none-any.whl → 3.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- eodag/api/core.py +347 -247
- eodag/api/product/_assets.py +44 -15
- eodag/api/product/_product.py +58 -47
- eodag/api/product/drivers/__init__.py +81 -4
- eodag/api/product/drivers/base.py +65 -4
- eodag/api/product/drivers/generic.py +65 -0
- eodag/api/product/drivers/sentinel1.py +97 -0
- eodag/api/product/drivers/sentinel2.py +95 -0
- eodag/api/product/metadata_mapping.py +129 -93
- eodag/api/search_result.py +28 -12
- eodag/cli.py +61 -24
- eodag/config.py +457 -167
- eodag/plugins/apis/base.py +10 -4
- eodag/plugins/apis/ecmwf.py +53 -23
- eodag/plugins/apis/usgs.py +41 -17
- eodag/plugins/authentication/aws_auth.py +30 -18
- eodag/plugins/authentication/base.py +14 -3
- eodag/plugins/authentication/generic.py +14 -3
- eodag/plugins/authentication/header.py +14 -6
- eodag/plugins/authentication/keycloak.py +44 -25
- eodag/plugins/authentication/oauth.py +18 -4
- eodag/plugins/authentication/openid_connect.py +192 -171
- eodag/plugins/authentication/qsauth.py +12 -4
- eodag/plugins/authentication/sas_auth.py +22 -5
- eodag/plugins/authentication/token.py +95 -17
- eodag/plugins/authentication/token_exchange.py +19 -19
- eodag/plugins/base.py +4 -4
- eodag/plugins/crunch/base.py +8 -5
- eodag/plugins/crunch/filter_date.py +9 -6
- eodag/plugins/crunch/filter_latest_intersect.py +9 -8
- eodag/plugins/crunch/filter_latest_tpl_name.py +8 -8
- eodag/plugins/crunch/filter_overlap.py +9 -11
- eodag/plugins/crunch/filter_property.py +10 -10
- eodag/plugins/download/aws.py +181 -105
- eodag/plugins/download/base.py +49 -67
- eodag/plugins/download/creodias_s3.py +40 -2
- eodag/plugins/download/http.py +247 -223
- eodag/plugins/download/s3rest.py +29 -28
- eodag/plugins/manager.py +176 -41
- eodag/plugins/search/__init__.py +6 -5
- eodag/plugins/search/base.py +123 -60
- eodag/plugins/search/build_search_result.py +1046 -355
- eodag/plugins/search/cop_marine.py +132 -39
- eodag/plugins/search/creodias_s3.py +19 -68
- eodag/plugins/search/csw.py +48 -8
- eodag/plugins/search/data_request_search.py +124 -23
- eodag/plugins/search/qssearch.py +531 -310
- eodag/plugins/search/stac_list_assets.py +85 -0
- eodag/plugins/search/static_stac_search.py +23 -24
- eodag/resources/ext_product_types.json +1 -1
- eodag/resources/product_types.yml +1295 -355
- eodag/resources/providers.yml +1819 -3010
- eodag/resources/stac.yml +3 -163
- eodag/resources/stac_api.yml +2 -2
- eodag/resources/user_conf_template.yml +115 -99
- eodag/rest/cache.py +2 -2
- eodag/rest/config.py +3 -4
- eodag/rest/constants.py +0 -1
- eodag/rest/core.py +157 -117
- eodag/rest/errors.py +181 -0
- eodag/rest/server.py +57 -339
- eodag/rest/stac.py +133 -581
- eodag/rest/types/collections_search.py +3 -3
- eodag/rest/types/eodag_search.py +41 -30
- eodag/rest/types/queryables.py +42 -32
- eodag/rest/types/stac_search.py +15 -16
- eodag/rest/utils/__init__.py +14 -21
- eodag/rest/utils/cql_evaluate.py +6 -6
- eodag/rest/utils/rfc3339.py +2 -2
- eodag/types/__init__.py +153 -32
- eodag/types/bbox.py +2 -2
- eodag/types/download_args.py +4 -4
- eodag/types/queryables.py +183 -73
- eodag/types/search_args.py +6 -6
- eodag/types/whoosh.py +127 -3
- eodag/utils/__init__.py +228 -106
- eodag/utils/exceptions.py +47 -26
- eodag/utils/import_system.py +2 -2
- eodag/utils/logging.py +37 -77
- eodag/utils/repr.py +65 -6
- eodag/utils/requests.py +13 -15
- eodag/utils/rest.py +2 -2
- eodag/utils/s3.py +231 -0
- eodag/utils/stac_reader.py +11 -11
- {eodag-3.0.0b3.dist-info → eodag-3.1.0.dist-info}/METADATA +81 -81
- eodag-3.1.0.dist-info/RECORD +113 -0
- {eodag-3.0.0b3.dist-info → eodag-3.1.0.dist-info}/WHEEL +1 -1
- {eodag-3.0.0b3.dist-info → eodag-3.1.0.dist-info}/entry_points.txt +5 -2
- eodag/resources/constraints/climate-dt.json +0 -13
- eodag/resources/constraints/extremes-dt.json +0 -8
- eodag/utils/constraints.py +0 -244
- eodag-3.0.0b3.dist-info/RECORD +0 -110
- {eodag-3.0.0b3.dist-info → eodag-3.1.0.dist-info}/LICENSE +0 -0
- {eodag-3.0.0b3.dist-info → eodag-3.1.0.dist-info}/top_level.txt +0 -0
|
@@ -0,0 +1,95 @@
|
|
|
1
|
+
# -*- coding: utf-8 -*-
|
|
2
|
+
# Copyright 2021, CS GROUP - France, http://www.c-s.fr
|
|
3
|
+
#
|
|
4
|
+
# This file is part of EODAG project
|
|
5
|
+
# https://www.github.com/CS-SI/EODAG
|
|
6
|
+
#
|
|
7
|
+
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
8
|
+
# you may not use this file except in compliance with the License.
|
|
9
|
+
# You may obtain a copy of the License at
|
|
10
|
+
#
|
|
11
|
+
# http://www.apache.org/licenses/LICENSE-2.0
|
|
12
|
+
#
|
|
13
|
+
# Unless required by applicable law or agreed to in writing, software
|
|
14
|
+
# distributed under the License is distributed on an "AS IS" BASIS,
|
|
15
|
+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
16
|
+
# See the License for the specific language governing permissions and
|
|
17
|
+
# limitations under the License.
|
|
18
|
+
from __future__ import annotations
|
|
19
|
+
|
|
20
|
+
import re
|
|
21
|
+
from typing import TYPE_CHECKING
|
|
22
|
+
|
|
23
|
+
from eodag.api.product.drivers.base import AssetPatterns, DatasetDriver
|
|
24
|
+
|
|
25
|
+
if TYPE_CHECKING:
|
|
26
|
+
from eodag.api.product._product import EOProduct
|
|
27
|
+
|
|
28
|
+
|
|
29
|
+
class Sentinel2Driver(DatasetDriver):
|
|
30
|
+
"""Driver for Sentinel2 products"""
|
|
31
|
+
|
|
32
|
+
#: Band keys associated with their default Ground Sampling Distance (GSD)
|
|
33
|
+
BANDS_DEFAULT_GSD = {
|
|
34
|
+
"10M": ("B02", "B03", "B04", "B08", "TCI"),
|
|
35
|
+
"20M": ("B05", "B06", "B07", "B11", "B12", "B8A"),
|
|
36
|
+
"60M": ("B01", "B09", "B10"),
|
|
37
|
+
}
|
|
38
|
+
|
|
39
|
+
#: list of patterns to match asset keys and roles
|
|
40
|
+
ASSET_KEYS_PATTERNS_ROLES: list[AssetPatterns] = [
|
|
41
|
+
# masks
|
|
42
|
+
{
|
|
43
|
+
"pattern": re.compile(r"^.*?(MSK_[^/\\]+)\.(?:jp2|tiff?)$", re.IGNORECASE),
|
|
44
|
+
"roles": ["data-mask"],
|
|
45
|
+
},
|
|
46
|
+
# visual
|
|
47
|
+
{
|
|
48
|
+
"pattern": re.compile(
|
|
49
|
+
r"^.*?(TCI)(_[0-9]+m)?\.(?:jp2|tiff?)$", re.IGNORECASE
|
|
50
|
+
),
|
|
51
|
+
"roles": ["visual"],
|
|
52
|
+
},
|
|
53
|
+
# bands
|
|
54
|
+
{
|
|
55
|
+
"pattern": re.compile(
|
|
56
|
+
r"^.*?([A-Z]+[0-9]*[A-Z]?)(_[0-9]+m)?\.(?:jp2|tiff?)$", re.IGNORECASE
|
|
57
|
+
),
|
|
58
|
+
"roles": ["data"],
|
|
59
|
+
},
|
|
60
|
+
# metadata
|
|
61
|
+
{
|
|
62
|
+
"pattern": re.compile(
|
|
63
|
+
r"^(?:.*[/\\])?([^/\\]+)(\.xml|\.xsd|\.safe|\.json)$", re.IGNORECASE
|
|
64
|
+
),
|
|
65
|
+
"roles": ["metadata"],
|
|
66
|
+
},
|
|
67
|
+
# thumbnail
|
|
68
|
+
{
|
|
69
|
+
"pattern": re.compile(
|
|
70
|
+
r"^(?:.*[/\\])?(thumbnail)(\.jpe?g|\.png)$", re.IGNORECASE
|
|
71
|
+
),
|
|
72
|
+
"roles": ["thumbnail"],
|
|
73
|
+
},
|
|
74
|
+
# quicklook
|
|
75
|
+
{
|
|
76
|
+
"pattern": re.compile(
|
|
77
|
+
r"^(?:.*[/\\])?[^/\\]+(-ql|preview|quick-?look)(\.jpe?g|\.png)$",
|
|
78
|
+
re.IGNORECASE,
|
|
79
|
+
),
|
|
80
|
+
"roles": ["overview"],
|
|
81
|
+
},
|
|
82
|
+
# default
|
|
83
|
+
{"pattern": re.compile(r"^(?:.*[/\\])?([^/\\]+)$"), "roles": ["auxiliary"]},
|
|
84
|
+
]
|
|
85
|
+
|
|
86
|
+
def _normalize_key(self, key: str, eo_product: EOProduct) -> str:
|
|
87
|
+
upper_key = key.upper()
|
|
88
|
+
# check if key matched any normalized
|
|
89
|
+
for res in self.BANDS_DEFAULT_GSD:
|
|
90
|
+
if res in upper_key:
|
|
91
|
+
for norm_key in self.BANDS_DEFAULT_GSD[res]:
|
|
92
|
+
if norm_key in upper_key:
|
|
93
|
+
return norm_key
|
|
94
|
+
|
|
95
|
+
return super()._normalize_key(key, eo_product)
|
|
@@ -23,24 +23,14 @@ import logging
|
|
|
23
23
|
import re
|
|
24
24
|
from datetime import datetime, timedelta
|
|
25
25
|
from string import Formatter
|
|
26
|
-
from typing import
|
|
27
|
-
TYPE_CHECKING,
|
|
28
|
-
Any,
|
|
29
|
-
AnyStr,
|
|
30
|
-
Callable,
|
|
31
|
-
Dict,
|
|
32
|
-
Iterator,
|
|
33
|
-
List,
|
|
34
|
-
Optional,
|
|
35
|
-
Tuple,
|
|
36
|
-
Union,
|
|
37
|
-
cast,
|
|
38
|
-
)
|
|
26
|
+
from typing import TYPE_CHECKING, Any, AnyStr, Callable, Iterator, Optional, Union, cast
|
|
39
27
|
|
|
40
28
|
import geojson
|
|
41
29
|
import orjson
|
|
42
30
|
import pyproj
|
|
31
|
+
import shapely
|
|
43
32
|
from dateutil.parser import isoparse
|
|
33
|
+
from dateutil.relativedelta import relativedelta
|
|
44
34
|
from dateutil.tz import UTC, tzutc
|
|
45
35
|
from jsonpath_ng.jsonpath import Child, JSONPath
|
|
46
36
|
from lxml import etree
|
|
@@ -86,8 +76,8 @@ DEFAULT_GEOMETRY = "POLYGON((180 -90, 180 90, -180 90, -180 -90, 180 -90))"
|
|
|
86
76
|
|
|
87
77
|
|
|
88
78
|
def get_metadata_path(
|
|
89
|
-
map_value: Union[str,
|
|
90
|
-
) ->
|
|
79
|
+
map_value: Union[str, list[str]],
|
|
80
|
+
) -> tuple[Union[list[str], None], str]:
|
|
91
81
|
"""Return the jsonpath or xpath to the value of a EO product metadata in a provider
|
|
92
82
|
search result.
|
|
93
83
|
|
|
@@ -135,12 +125,12 @@ def get_metadata_path(
|
|
|
135
125
|
return None, path
|
|
136
126
|
|
|
137
127
|
|
|
138
|
-
def get_metadata_path_value(map_value: Union[str,
|
|
128
|
+
def get_metadata_path_value(map_value: Union[str, list[str]]) -> str:
|
|
139
129
|
"""Get raw metadata path without converter"""
|
|
140
130
|
return map_value[1] if isinstance(map_value, list) else map_value
|
|
141
131
|
|
|
142
132
|
|
|
143
|
-
def get_search_param(map_value:
|
|
133
|
+
def get_search_param(map_value: list[str]) -> str:
|
|
144
134
|
"""See :func:`~eodag.api.product.metadata_mapping.get_metadata_path`
|
|
145
135
|
|
|
146
136
|
:param map_value: The value originating from the definition of `metadata_mapping`
|
|
@@ -152,7 +142,7 @@ def get_search_param(map_value: List[str]) -> str:
|
|
|
152
142
|
|
|
153
143
|
|
|
154
144
|
def format_metadata(search_param: str, *args: Any, **kwargs: Any) -> str:
|
|
155
|
-
"""Format a string of form {<field_name>#<conversion_function>}
|
|
145
|
+
"""Format a string of form ``{<field_name>#<conversion_function>}``
|
|
156
146
|
|
|
157
147
|
The currently understood converters are:
|
|
158
148
|
- ``datetime_to_timestamp_milliseconds``: converts a utc date string to a timestamp in
|
|
@@ -178,6 +168,8 @@ def format_metadata(search_param: str, *args: Any, **kwargs: Any) -> str:
|
|
|
178
168
|
- ``recursive_sub_str``: recursively substitue in the structure (e.g. dict)
|
|
179
169
|
values matching a regex
|
|
180
170
|
- ``slice_str``: slice a string (equivalent to s[start, end, step])
|
|
171
|
+
- ``to_lower``: Convert a string to lowercase
|
|
172
|
+
- ``to_upper``: Convert a string to uppercase
|
|
181
173
|
- ``fake_l2a_title_from_l1c``: used to generate SAFE format metadata for data from AWS
|
|
182
174
|
- ``s2msil2a_title_to_aws_productinfo``: used to generate SAFE format metadata for data from AWS
|
|
183
175
|
- ``split_cop_dem_id``: get the bbox by splitting the product id
|
|
@@ -331,7 +323,7 @@ def format_metadata(search_param: str, *args: Any, **kwargs: Any) -> str:
|
|
|
331
323
|
return wkt_value
|
|
332
324
|
|
|
333
325
|
@staticmethod
|
|
334
|
-
def convert_to_bounds_lists(input_geom: BaseGeometry) ->
|
|
326
|
+
def convert_to_bounds_lists(input_geom: BaseGeometry) -> list[list[float]]:
|
|
335
327
|
if isinstance(input_geom, MultiPolygon):
|
|
336
328
|
geoms = [geom for geom in input_geom.geoms]
|
|
337
329
|
# sort with larger one at first (stac-browser only plots first one)
|
|
@@ -341,7 +333,7 @@ def format_metadata(search_param: str, *args: Any, **kwargs: Any) -> str:
|
|
|
341
333
|
return [list(input_geom.bounds[0:4])]
|
|
342
334
|
|
|
343
335
|
@staticmethod
|
|
344
|
-
def convert_to_bounds(input_geom_unformatted: Any) ->
|
|
336
|
+
def convert_to_bounds(input_geom_unformatted: Any) -> list[float]:
|
|
345
337
|
input_geom = get_geometry_from_various(geometry=input_geom_unformatted)
|
|
346
338
|
if isinstance(input_geom, MultiPolygon):
|
|
347
339
|
geoms = [geom for geom in input_geom.geoms]
|
|
@@ -352,16 +344,18 @@ def format_metadata(search_param: str, *args: Any, **kwargs: Any) -> str:
|
|
|
352
344
|
max_lon = -180
|
|
353
345
|
max_lat = -90
|
|
354
346
|
for geom in geoms:
|
|
355
|
-
min_lon = min(min_lon, geom.
|
|
356
|
-
min_lat = min(min_lat, geom.
|
|
357
|
-
max_lon = max(max_lon, geom.
|
|
358
|
-
max_lat = max(max_lat, geom.
|
|
347
|
+
min_lon = min(min_lon, geom.bounds[0])
|
|
348
|
+
min_lat = min(min_lat, geom.bounds[1])
|
|
349
|
+
max_lon = max(max_lon, geom.bounds[2])
|
|
350
|
+
max_lat = max(max_lat, geom.bounds[3])
|
|
359
351
|
return [min_lon, min_lat, max_lon, max_lat]
|
|
360
352
|
else:
|
|
361
353
|
return list(input_geom.bounds[0:4])
|
|
362
354
|
|
|
363
355
|
@staticmethod
|
|
364
|
-
def convert_to_nwse_bounds(input_geom: BaseGeometry) ->
|
|
356
|
+
def convert_to_nwse_bounds(input_geom: BaseGeometry) -> list[float]:
|
|
357
|
+
if isinstance(input_geom, str):
|
|
358
|
+
input_geom = shapely.wkt.loads(input_geom)
|
|
365
359
|
return list(input_geom.bounds[-1:] + input_geom.bounds[:-1])
|
|
366
360
|
|
|
367
361
|
@staticmethod
|
|
@@ -373,8 +367,8 @@ def format_metadata(search_param: str, *args: Any, **kwargs: Any) -> str:
|
|
|
373
367
|
)
|
|
374
368
|
|
|
375
369
|
@staticmethod
|
|
376
|
-
def convert_to_geojson(
|
|
377
|
-
return geojson.dumps(
|
|
370
|
+
def convert_to_geojson(value: Any) -> str:
|
|
371
|
+
return geojson.dumps(value)
|
|
378
372
|
|
|
379
373
|
@staticmethod
|
|
380
374
|
def convert_from_ewkt(ewkt_string: str) -> Union[BaseGeometry, str]:
|
|
@@ -443,7 +437,7 @@ def format_metadata(search_param: str, *args: Any, **kwargs: Any) -> str:
|
|
|
443
437
|
else:
|
|
444
438
|
yield e
|
|
445
439
|
|
|
446
|
-
polygons_list:
|
|
440
|
+
polygons_list: list[Polygon] = []
|
|
447
441
|
for elem in flatten_elements(georss[0]):
|
|
448
442
|
coords_list = elem.text.split()
|
|
449
443
|
polygon_args = [
|
|
@@ -468,7 +462,7 @@ def format_metadata(search_param: str, *args: Any, **kwargs: Any) -> str:
|
|
|
468
462
|
@staticmethod
|
|
469
463
|
def convert_to_longitude_latitude(
|
|
470
464
|
input_geom_unformatted: Any,
|
|
471
|
-
) ->
|
|
465
|
+
) -> dict[str, float]:
|
|
472
466
|
bounds = MetadataFormatter.convert_to_bounds(input_geom_unformatted)
|
|
473
467
|
lon = (bounds[0] + bounds[2]) / 2
|
|
474
468
|
lat = (bounds[1] + bounds[3]) / 2
|
|
@@ -502,14 +496,21 @@ def format_metadata(search_param: str, *args: Any, **kwargs: Any) -> str:
|
|
|
502
496
|
return NOT_AVAILABLE
|
|
503
497
|
|
|
504
498
|
@staticmethod
|
|
505
|
-
def convert_replace_str(
|
|
499
|
+
def convert_replace_str(value: Any, args: str) -> str:
|
|
500
|
+
if isinstance(value, dict):
|
|
501
|
+
value = MetadataFormatter.convert_to_geojson(value)
|
|
502
|
+
elif not isinstance(value, str):
|
|
503
|
+
raise TypeError(
|
|
504
|
+
f"convert_replace_str expects a string or a dict (apply to_geojson). Got {type(value)}"
|
|
505
|
+
)
|
|
506
|
+
|
|
506
507
|
old, new = ast.literal_eval(args)
|
|
507
|
-
return re.sub(old, new,
|
|
508
|
+
return re.sub(old, new, value)
|
|
508
509
|
|
|
509
510
|
@staticmethod
|
|
510
511
|
def convert_recursive_sub_str(
|
|
511
|
-
input_obj: Union[
|
|
512
|
-
) -> Union[
|
|
512
|
+
input_obj: Union[dict[Any, Any], list[Any]], args: str
|
|
513
|
+
) -> Union[dict[Any, Any], list[Any]]:
|
|
513
514
|
old, new = ast.literal_eval(args)
|
|
514
515
|
return items_recursive_apply(
|
|
515
516
|
input_obj,
|
|
@@ -519,8 +520,8 @@ def format_metadata(search_param: str, *args: Any, **kwargs: Any) -> str:
|
|
|
519
520
|
|
|
520
521
|
@staticmethod
|
|
521
522
|
def convert_dict_update(
|
|
522
|
-
input_dict:
|
|
523
|
-
) ->
|
|
523
|
+
input_dict: dict[Any, Any], args: str
|
|
524
|
+
) -> dict[Any, Any]:
|
|
524
525
|
"""Converts"""
|
|
525
526
|
new_items_list = ast.literal_eval(args)
|
|
526
527
|
|
|
@@ -530,8 +531,8 @@ def format_metadata(search_param: str, *args: Any, **kwargs: Any) -> str:
|
|
|
530
531
|
|
|
531
532
|
@staticmethod
|
|
532
533
|
def convert_dict_filter(
|
|
533
|
-
input_dict:
|
|
534
|
-
) ->
|
|
534
|
+
input_dict: dict[Any, Any], jsonpath_filter_str: str
|
|
535
|
+
) -> dict[Any, Any]:
|
|
535
536
|
"""Fitlers dict items using jsonpath"""
|
|
536
537
|
|
|
537
538
|
jsonpath_filter = string_to_jsonpath(jsonpath_filter_str, force=True)
|
|
@@ -557,6 +558,16 @@ def format_metadata(search_param: str, *args: Any, **kwargs: Any) -> str:
|
|
|
557
558
|
]
|
|
558
559
|
return string[cmin:cmax:cstep]
|
|
559
560
|
|
|
561
|
+
@staticmethod
|
|
562
|
+
def convert_to_lower(string: str) -> str:
|
|
563
|
+
"""Convert a string to lowercase."""
|
|
564
|
+
return string.lower()
|
|
565
|
+
|
|
566
|
+
@staticmethod
|
|
567
|
+
def convert_to_upper(string: str) -> str:
|
|
568
|
+
"""Convert a string to uppercase."""
|
|
569
|
+
return string.upper()
|
|
570
|
+
|
|
560
571
|
@staticmethod
|
|
561
572
|
def convert_fake_l2a_title_from_l1c(string: str) -> str:
|
|
562
573
|
id_regex = re.compile(
|
|
@@ -600,8 +611,8 @@ def format_metadata(search_param: str, *args: Any, **kwargs: Any) -> str:
|
|
|
600
611
|
return NOT_AVAILABLE
|
|
601
612
|
|
|
602
613
|
@staticmethod
|
|
603
|
-
def convert_split_id_into_s1_params(product_id: str) ->
|
|
604
|
-
parts:
|
|
614
|
+
def convert_split_id_into_s1_params(product_id: str) -> dict[str, str]:
|
|
615
|
+
parts: list[str] = re.split(r"_(?!_)", product_id)
|
|
605
616
|
if len(parts) < 9:
|
|
606
617
|
logger.error(
|
|
607
618
|
"id %s does not match expected Sentinel-1 id format", product_id
|
|
@@ -635,8 +646,8 @@ def format_metadata(search_param: str, *args: Any, **kwargs: Any) -> str:
|
|
|
635
646
|
return params
|
|
636
647
|
|
|
637
648
|
@staticmethod
|
|
638
|
-
def convert_split_id_into_s3_params(product_id: str) ->
|
|
639
|
-
parts:
|
|
649
|
+
def convert_split_id_into_s3_params(product_id: str) -> dict[str, str]:
|
|
650
|
+
parts: list[str] = re.split(r"_(?!_)", product_id)
|
|
640
651
|
params = {"productType": product_id[4:15]}
|
|
641
652
|
dates = re.findall("[0-9]{8}T[0-9]{6}", product_id)
|
|
642
653
|
start_date = datetime.strptime(dates[0], "%Y%m%dT%H%M%S") - timedelta(
|
|
@@ -652,8 +663,8 @@ def format_metadata(search_param: str, *args: Any, **kwargs: Any) -> str:
|
|
|
652
663
|
return params
|
|
653
664
|
|
|
654
665
|
@staticmethod
|
|
655
|
-
def convert_split_id_into_s5p_params(product_id: str) ->
|
|
656
|
-
parts:
|
|
666
|
+
def convert_split_id_into_s5p_params(product_id: str) -> dict[str, str]:
|
|
667
|
+
parts: list[str] = re.split(r"_(?!_)", product_id)
|
|
657
668
|
params = {
|
|
658
669
|
"productType": product_id[9:19],
|
|
659
670
|
"processingMode": parts[1],
|
|
@@ -670,7 +681,7 @@ def format_metadata(search_param: str, *args: Any, **kwargs: Any) -> str:
|
|
|
670
681
|
return params
|
|
671
682
|
|
|
672
683
|
@staticmethod
|
|
673
|
-
def convert_split_cop_dem_id(product_id: str) ->
|
|
684
|
+
def convert_split_cop_dem_id(product_id: str) -> list[int]:
|
|
674
685
|
parts = product_id.split("_")
|
|
675
686
|
lattitude = parts[3]
|
|
676
687
|
longitude = parts[5]
|
|
@@ -709,7 +720,7 @@ def format_metadata(search_param: str, *args: Any, **kwargs: Any) -> str:
|
|
|
709
720
|
@staticmethod
|
|
710
721
|
def convert_to_datetime_dict(
|
|
711
722
|
date: str, format: str
|
|
712
|
-
) ->
|
|
723
|
+
) -> dict[str, Union[list[str], str]]:
|
|
713
724
|
"""Convert a date (str) to a dictionary where values are in the format given in argument
|
|
714
725
|
|
|
715
726
|
date == "2021-04-21T18:27:19.123Z" and format == "list" => {
|
|
@@ -761,7 +772,7 @@ def format_metadata(search_param: str, *args: Any, **kwargs: Any) -> str:
|
|
|
761
772
|
@staticmethod
|
|
762
773
|
def convert_interval_to_datetime_dict(
|
|
763
774
|
date: str, separator: str = "/"
|
|
764
|
-
) ->
|
|
775
|
+
) -> dict[str, list[str]]:
|
|
765
776
|
"""Convert a date interval ('/' separated str) to a dictionary where values are lists
|
|
766
777
|
|
|
767
778
|
date == "2021-04-21/2021-04-22" => {
|
|
@@ -801,7 +812,7 @@ def format_metadata(search_param: str, *args: Any, **kwargs: Any) -> str:
|
|
|
801
812
|
}
|
|
802
813
|
|
|
803
814
|
@staticmethod
|
|
804
|
-
def convert_get_ecmwf_time(date: str) ->
|
|
815
|
+
def convert_get_ecmwf_time(date: str) -> list[str]:
|
|
805
816
|
"""Get the time of a date (str) in the ECMWF format (["HH:00"])
|
|
806
817
|
|
|
807
818
|
"2021-04-21T18:27:19.123Z" => ["18:00"]
|
|
@@ -831,9 +842,9 @@ def format_metadata(search_param: str, *args: Any, **kwargs: Any) -> str:
|
|
|
831
842
|
def convert_get_hydrological_year(date: str):
|
|
832
843
|
utc_date = MetadataFormatter.convert_to_iso_utc_datetime(date)
|
|
833
844
|
date_object = datetime.strptime(utc_date, "%Y-%m-%dT%H:%M:%S.%fZ")
|
|
834
|
-
date_object_second_year = date_object +
|
|
845
|
+
date_object_second_year = date_object + relativedelta(years=1)
|
|
835
846
|
return [
|
|
836
|
-
f
|
|
847
|
+
f"{date_object.strftime('%Y')}_{date_object_second_year.strftime('%y')}"
|
|
837
848
|
]
|
|
838
849
|
|
|
839
850
|
@staticmethod
|
|
@@ -845,8 +856,8 @@ def format_metadata(search_param: str, *args: Any, **kwargs: Any) -> str:
|
|
|
845
856
|
|
|
846
857
|
@staticmethod
|
|
847
858
|
def convert_assets_list_to_dict(
|
|
848
|
-
assets_list:
|
|
849
|
-
) ->
|
|
859
|
+
assets_list: list[dict[str, str]], asset_name_key: str = "title"
|
|
860
|
+
) -> dict[str, dict[str, str]]:
|
|
850
861
|
"""Convert a list of assets to a dictionary where keys represent
|
|
851
862
|
name of assets and are found among values of asset dictionaries.
|
|
852
863
|
|
|
@@ -873,8 +884,8 @@ def format_metadata(search_param: str, *args: Any, **kwargs: Any) -> str:
|
|
|
873
884
|
"asset3": {"href": "qux", "title": "qux-title", "name": "asset3"},
|
|
874
885
|
}
|
|
875
886
|
"""
|
|
876
|
-
asset_names:
|
|
877
|
-
assets_dict:
|
|
887
|
+
asset_names: list[str] = []
|
|
888
|
+
assets_dict: dict[str, dict[str, str]] = {}
|
|
878
889
|
|
|
879
890
|
for asset in assets_list:
|
|
880
891
|
asset_name = asset[asset_name_key]
|
|
@@ -883,7 +894,7 @@ def format_metadata(search_param: str, *args: Any, **kwargs: Any) -> str:
|
|
|
883
894
|
|
|
884
895
|
# we only keep the equivalent of the path basename in the case where the
|
|
885
896
|
# asset name has a path pattern and this basename is only found once
|
|
886
|
-
immutable_asset_indexes:
|
|
897
|
+
immutable_asset_indexes: list[int] = []
|
|
887
898
|
for i, asset_name in enumerate(asset_names):
|
|
888
899
|
if i in immutable_asset_indexes:
|
|
889
900
|
continue
|
|
@@ -901,20 +912,18 @@ def format_metadata(search_param: str, *args: Any, **kwargs: Any) -> str:
|
|
|
901
912
|
return assets_dict
|
|
902
913
|
|
|
903
914
|
# if stac extension colon separator `:` is in search params, parse it to prevent issues with vformat
|
|
904
|
-
if re.search(r"{[
|
|
905
|
-
search_param = re.sub(
|
|
906
|
-
r"{([a-zA-Z0-9_-]*):([a-zA-Z0-9_-]*)}", r"{\1_COLON_\2}", search_param
|
|
907
|
-
)
|
|
915
|
+
if re.search(r"{[\w-]*:[\w#-]*}", search_param):
|
|
916
|
+
search_param = re.sub(r"{([\w-]*):([\w#-]*)}", r"{\1_COLON_\2}", search_param)
|
|
908
917
|
kwargs = {k.replace(":", "_COLON_"): v for k, v in kwargs.items()}
|
|
909
918
|
|
|
910
919
|
return MetadataFormatter().vformat(search_param, args, kwargs)
|
|
911
920
|
|
|
912
921
|
|
|
913
922
|
def properties_from_json(
|
|
914
|
-
json:
|
|
915
|
-
mapping:
|
|
916
|
-
discovery_config: Optional[
|
|
917
|
-
) ->
|
|
923
|
+
json: dict[str, Any],
|
|
924
|
+
mapping: dict[str, Any],
|
|
925
|
+
discovery_config: Optional[dict[str, Any]] = None,
|
|
926
|
+
) -> dict[str, Any]:
|
|
918
927
|
"""Extract properties from a provider json result.
|
|
919
928
|
|
|
920
929
|
:param json: The representation of a provider result as a json object
|
|
@@ -927,7 +936,7 @@ def properties_from_json(
|
|
|
927
936
|
`discovery_path` (String representation of jsonpath)
|
|
928
937
|
:returns: The metadata of the :class:`~eodag.api.product._product.EOProduct`
|
|
929
938
|
"""
|
|
930
|
-
properties:
|
|
939
|
+
properties: dict[str, Any] = {}
|
|
931
940
|
templates = {}
|
|
932
941
|
used_jsonpaths = []
|
|
933
942
|
for metadata, value in mapping.items():
|
|
@@ -974,10 +983,24 @@ def properties_from_json(
|
|
|
974
983
|
if re.search(r"({[^{}:]+})+", conversion_or_none):
|
|
975
984
|
conversion_or_none = conversion_or_none.format(**properties)
|
|
976
985
|
|
|
977
|
-
|
|
978
|
-
|
|
979
|
-
|
|
980
|
-
|
|
986
|
+
if extracted_value == NOT_AVAILABLE:
|
|
987
|
+
# try if value can be formatted even if it is not available
|
|
988
|
+
try:
|
|
989
|
+
properties[metadata] = format_metadata(
|
|
990
|
+
"{%s%s%s}" % (metadata, SEP, conversion_or_none),
|
|
991
|
+
**{metadata: extracted_value},
|
|
992
|
+
)
|
|
993
|
+
except ValueError:
|
|
994
|
+
logger.debug(
|
|
995
|
+
f"{metadata}: {extracted_value} could not be formatted with {conversion_or_none}"
|
|
996
|
+
)
|
|
997
|
+
continue
|
|
998
|
+
else:
|
|
999
|
+
# in this case formatting should work, otherwise something is wrong in the mapping
|
|
1000
|
+
properties[metadata] = format_metadata(
|
|
1001
|
+
"{%s%s%s}" % (metadata, SEP, conversion_or_none),
|
|
1002
|
+
**{metadata: extracted_value},
|
|
1003
|
+
)
|
|
981
1004
|
# properties as python objects when possible (format_metadata returns only strings)
|
|
982
1005
|
try:
|
|
983
1006
|
properties[metadata] = ast.literal_eval(properties[metadata])
|
|
@@ -1057,8 +1080,8 @@ def properties_from_xml(
|
|
|
1057
1080
|
xml_as_text: AnyStr,
|
|
1058
1081
|
mapping: Any,
|
|
1059
1082
|
empty_ns_prefix: str = "ns",
|
|
1060
|
-
discovery_config: Optional[
|
|
1061
|
-
) ->
|
|
1083
|
+
discovery_config: Optional[dict[str, Any]] = None,
|
|
1084
|
+
) -> dict[str, Any]:
|
|
1062
1085
|
"""Extract properties from a provider xml result.
|
|
1063
1086
|
|
|
1064
1087
|
:param xml_as_text: The representation of a provider result as xml
|
|
@@ -1076,7 +1099,7 @@ def properties_from_xml(
|
|
|
1076
1099
|
`discovery_path` (String representation of xpath)
|
|
1077
1100
|
:returns: the metadata of the :class:`~eodag.api.product._product.EOProduct`
|
|
1078
1101
|
"""
|
|
1079
|
-
properties:
|
|
1102
|
+
properties: dict[str, Any] = {}
|
|
1080
1103
|
templates = {}
|
|
1081
1104
|
used_xpaths = []
|
|
1082
1105
|
root = etree.XML(xml_as_text)
|
|
@@ -1204,11 +1227,11 @@ def properties_from_xml(
|
|
|
1204
1227
|
|
|
1205
1228
|
|
|
1206
1229
|
def mtd_cfg_as_conversion_and_querypath(
|
|
1207
|
-
src_dict:
|
|
1208
|
-
dest_dict:
|
|
1230
|
+
src_dict: dict[str, Any],
|
|
1231
|
+
dest_dict: dict[str, Any] = {},
|
|
1209
1232
|
result_type: str = "json",
|
|
1210
|
-
) ->
|
|
1211
|
-
"""Metadata configuration dictionary to querypath with conversion
|
|
1233
|
+
) -> dict[str, Any]:
|
|
1234
|
+
"""Metadata configuration dictionary to querypath with conversion dictionary
|
|
1212
1235
|
Transform every src_dict value from jsonpath_str to tuple `(conversion, jsonpath_object)`
|
|
1213
1236
|
or from xpath_str to tuple `(conversion, xpath_str)`
|
|
1214
1237
|
|
|
@@ -1255,8 +1278,8 @@ def mtd_cfg_as_conversion_and_querypath(
|
|
|
1255
1278
|
|
|
1256
1279
|
|
|
1257
1280
|
def format_query_params(
|
|
1258
|
-
product_type: str, config: PluginConfig, query_dict:
|
|
1259
|
-
) ->
|
|
1281
|
+
product_type: str, config: PluginConfig, query_dict: dict[str, Any]
|
|
1282
|
+
) -> dict[str, Any]:
|
|
1260
1283
|
"""format the search parameters to query parameters"""
|
|
1261
1284
|
if "raise_errors" in query_dict.keys():
|
|
1262
1285
|
del query_dict["raise_errors"]
|
|
@@ -1268,7 +1291,7 @@ def format_query_params(
|
|
|
1268
1291
|
**config.products.get(product_type, {}).get("metadata_mapping", {}),
|
|
1269
1292
|
)
|
|
1270
1293
|
|
|
1271
|
-
query_params:
|
|
1294
|
+
query_params: dict[str, Any] = {}
|
|
1272
1295
|
# Get all the search parameters that are recognised as queryables by the
|
|
1273
1296
|
# provider (they appear in the queryables dictionary)
|
|
1274
1297
|
queryables = _get_queryables(query_dict, config, product_type_metadata_mapping)
|
|
@@ -1298,8 +1321,8 @@ def format_query_params(
|
|
|
1298
1321
|
query_params[eodag_search_key] = formatted_query_param
|
|
1299
1322
|
else:
|
|
1300
1323
|
provider_search_key, provider_value = parts
|
|
1301
|
-
query_params
|
|
1302
|
-
|
|
1324
|
+
query_params[provider_search_key] = format_metadata(
|
|
1325
|
+
provider_value, product_type, **query_dict
|
|
1303
1326
|
)
|
|
1304
1327
|
else:
|
|
1305
1328
|
query_params[provider_search_key] = user_input
|
|
@@ -1358,10 +1381,10 @@ def _resolve_hashes(formatted_query_param: str) -> str:
|
|
|
1358
1381
|
|
|
1359
1382
|
|
|
1360
1383
|
def _format_free_text_search(
|
|
1361
|
-
config: PluginConfig, metadata_mapping:
|
|
1362
|
-
) ->
|
|
1384
|
+
config: PluginConfig, metadata_mapping: dict[str, Any], **kwargs: Any
|
|
1385
|
+
) -> dict[str, Any]:
|
|
1363
1386
|
"""Build the free text search parameter using the search parameters"""
|
|
1364
|
-
query_params:
|
|
1387
|
+
query_params: dict[str, Any] = {}
|
|
1365
1388
|
if not getattr(config, "free_text_search_operations", None):
|
|
1366
1389
|
return query_params
|
|
1367
1390
|
for param, operations_config in config.free_text_search_operations.items():
|
|
@@ -1400,13 +1423,13 @@ def _format_free_text_search(
|
|
|
1400
1423
|
|
|
1401
1424
|
|
|
1402
1425
|
def _get_queryables(
|
|
1403
|
-
search_params:
|
|
1426
|
+
search_params: dict[str, Any],
|
|
1404
1427
|
config: PluginConfig,
|
|
1405
|
-
metadata_mapping:
|
|
1406
|
-
) ->
|
|
1428
|
+
metadata_mapping: dict[str, Any],
|
|
1429
|
+
) -> dict[str, Any]:
|
|
1407
1430
|
"""Retrieve the metadata mappings that are query-able"""
|
|
1408
1431
|
logger.debug("Retrieving queryable metadata from metadata_mapping")
|
|
1409
|
-
queryables:
|
|
1432
|
+
queryables: dict[str, Any] = {}
|
|
1410
1433
|
for eodag_search_key, user_input in search_params.items():
|
|
1411
1434
|
if user_input is not None:
|
|
1412
1435
|
md_mapping = metadata_mapping.get(eodag_search_key, (None, NOT_MAPPED))
|
|
@@ -1453,7 +1476,7 @@ def _get_queryables(
|
|
|
1453
1476
|
|
|
1454
1477
|
|
|
1455
1478
|
def get_queryable_from_provider(
|
|
1456
|
-
provider_queryable: str, metadata_mapping:
|
|
1479
|
+
provider_queryable: str, metadata_mapping: dict[str, Union[str, list[str]]]
|
|
1457
1480
|
) -> Optional[str]:
|
|
1458
1481
|
"""Get EODAG configured queryable parameter from provider queryable parameter
|
|
1459
1482
|
|
|
@@ -1461,15 +1484,27 @@ def get_queryable_from_provider(
|
|
|
1461
1484
|
:param metadata_mapping: metadata-mapping configuration
|
|
1462
1485
|
:returns: EODAG configured queryable parameter or None
|
|
1463
1486
|
"""
|
|
1464
|
-
pattern = rf"\
|
|
1487
|
+
pattern = rf"\"{provider_queryable}\""
|
|
1488
|
+
# if 1:1 mapping exists privilege this one instead of other mapping
|
|
1489
|
+
# e.g. provider queryable = year -> use year and not date in which year also appears
|
|
1490
|
+
mapping_values = [
|
|
1491
|
+
v[0] if isinstance(v, list) else "" for v in metadata_mapping.values()
|
|
1492
|
+
]
|
|
1493
|
+
if provider_queryable in mapping_values:
|
|
1494
|
+
ind = mapping_values.index(provider_queryable)
|
|
1495
|
+
return Queryables.get_queryable_from_alias(list(metadata_mapping.keys())[ind])
|
|
1465
1496
|
for param, param_conf in metadata_mapping.items():
|
|
1466
|
-
if
|
|
1497
|
+
if (
|
|
1498
|
+
isinstance(param_conf, list)
|
|
1499
|
+
and param_conf[0]
|
|
1500
|
+
and re.search(pattern, param_conf[0])
|
|
1501
|
+
):
|
|
1467
1502
|
return Queryables.get_queryable_from_alias(param)
|
|
1468
1503
|
return None
|
|
1469
1504
|
|
|
1470
1505
|
|
|
1471
1506
|
def get_provider_queryable_path(
|
|
1472
|
-
queryable: str, metadata_mapping:
|
|
1507
|
+
queryable: str, metadata_mapping: dict[str, Union[str, list[str]]]
|
|
1473
1508
|
) -> Optional[str]:
|
|
1474
1509
|
"""Get EODAG configured queryable path from its parameter
|
|
1475
1510
|
|
|
@@ -1486,10 +1521,11 @@ def get_provider_queryable_path(
|
|
|
1486
1521
|
|
|
1487
1522
|
def get_provider_queryable_key(
|
|
1488
1523
|
eodag_key: str,
|
|
1489
|
-
provider_queryables:
|
|
1490
|
-
metadata_mapping:
|
|
1524
|
+
provider_queryables: dict[str, Any],
|
|
1525
|
+
metadata_mapping: dict[str, Union[list[Any], str]],
|
|
1491
1526
|
) -> str:
|
|
1492
|
-
"""
|
|
1527
|
+
"""Finds the provider queryable corresponding to the given eodag key based on the metadata mapping
|
|
1528
|
+
|
|
1493
1529
|
:param eodag_key: key in eodag
|
|
1494
1530
|
:param provider_queryables: queryables returned from the provider
|
|
1495
1531
|
:param metadata_mapping: metadata mapping from which the keys are retrieved
|