eodag 3.0.1__py3-none-any.whl → 3.1.0b2__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- eodag/api/core.py +164 -127
- eodag/api/product/_assets.py +11 -11
- eodag/api/product/_product.py +45 -30
- eodag/api/product/drivers/__init__.py +81 -4
- eodag/api/product/drivers/base.py +65 -4
- eodag/api/product/drivers/generic.py +65 -0
- eodag/api/product/drivers/sentinel1.py +97 -0
- eodag/api/product/drivers/sentinel2.py +95 -0
- eodag/api/product/metadata_mapping.py +101 -85
- eodag/api/search_result.py +13 -23
- eodag/cli.py +26 -5
- eodag/config.py +78 -81
- eodag/plugins/apis/base.py +1 -1
- eodag/plugins/apis/ecmwf.py +46 -22
- eodag/plugins/apis/usgs.py +16 -15
- eodag/plugins/authentication/aws_auth.py +16 -13
- eodag/plugins/authentication/base.py +5 -3
- eodag/plugins/authentication/header.py +3 -3
- eodag/plugins/authentication/keycloak.py +4 -4
- eodag/plugins/authentication/oauth.py +7 -3
- eodag/plugins/authentication/openid_connect.py +16 -16
- eodag/plugins/authentication/sas_auth.py +4 -4
- eodag/plugins/authentication/token.py +41 -10
- eodag/plugins/authentication/token_exchange.py +1 -1
- eodag/plugins/base.py +4 -4
- eodag/plugins/crunch/base.py +4 -4
- eodag/plugins/crunch/filter_date.py +4 -4
- eodag/plugins/crunch/filter_latest_intersect.py +6 -6
- eodag/plugins/crunch/filter_latest_tpl_name.py +7 -7
- eodag/plugins/crunch/filter_overlap.py +4 -4
- eodag/plugins/crunch/filter_property.py +6 -7
- eodag/plugins/download/aws.py +58 -78
- eodag/plugins/download/base.py +38 -56
- eodag/plugins/download/creodias_s3.py +29 -0
- eodag/plugins/download/http.py +173 -183
- eodag/plugins/download/s3rest.py +10 -11
- eodag/plugins/manager.py +10 -20
- eodag/plugins/search/__init__.py +6 -5
- eodag/plugins/search/base.py +87 -44
- eodag/plugins/search/build_search_result.py +1067 -329
- eodag/plugins/search/cop_marine.py +22 -12
- eodag/plugins/search/creodias_s3.py +9 -73
- eodag/plugins/search/csw.py +11 -11
- eodag/plugins/search/data_request_search.py +16 -15
- eodag/plugins/search/qssearch.py +103 -187
- eodag/plugins/search/stac_list_assets.py +85 -0
- eodag/plugins/search/static_stac_search.py +3 -3
- eodag/resources/ext_product_types.json +1 -1
- eodag/resources/product_types.yml +663 -304
- eodag/resources/providers.yml +823 -1749
- eodag/resources/stac_api.yml +2 -2
- eodag/resources/user_conf_template.yml +11 -0
- eodag/rest/cache.py +2 -2
- eodag/rest/config.py +3 -3
- eodag/rest/core.py +112 -82
- eodag/rest/errors.py +5 -5
- eodag/rest/server.py +33 -14
- eodag/rest/stac.py +40 -38
- eodag/rest/types/collections_search.py +3 -3
- eodag/rest/types/eodag_search.py +29 -23
- eodag/rest/types/queryables.py +15 -16
- eodag/rest/types/stac_search.py +15 -25
- eodag/rest/utils/__init__.py +14 -21
- eodag/rest/utils/cql_evaluate.py +6 -6
- eodag/rest/utils/rfc3339.py +2 -2
- eodag/types/__init__.py +75 -28
- eodag/types/bbox.py +2 -2
- eodag/types/download_args.py +3 -3
- eodag/types/queryables.py +183 -72
- eodag/types/search_args.py +4 -4
- eodag/types/whoosh.py +127 -3
- eodag/utils/__init__.py +152 -50
- eodag/utils/exceptions.py +28 -21
- eodag/utils/import_system.py +2 -2
- eodag/utils/repr.py +65 -6
- eodag/utils/requests.py +13 -13
- eodag/utils/rest.py +2 -2
- eodag/utils/s3.py +208 -0
- eodag/utils/stac_reader.py +10 -10
- {eodag-3.0.1.dist-info → eodag-3.1.0b2.dist-info}/METADATA +77 -76
- eodag-3.1.0b2.dist-info/RECORD +113 -0
- {eodag-3.0.1.dist-info → eodag-3.1.0b2.dist-info}/WHEEL +1 -1
- {eodag-3.0.1.dist-info → eodag-3.1.0b2.dist-info}/entry_points.txt +4 -2
- eodag/utils/constraints.py +0 -244
- eodag-3.0.1.dist-info/RECORD +0 -109
- {eodag-3.0.1.dist-info → eodag-3.1.0b2.dist-info}/LICENSE +0 -0
- {eodag-3.0.1.dist-info → eodag-3.1.0b2.dist-info}/top_level.txt +0 -0
|
@@ -17,21 +17,13 @@
|
|
|
17
17
|
# limitations under the License.
|
|
18
18
|
from __future__ import annotations
|
|
19
19
|
|
|
20
|
+
import functools
|
|
20
21
|
import hashlib
|
|
21
22
|
import logging
|
|
22
|
-
|
|
23
|
-
from
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
Any,
|
|
27
|
-
Dict,
|
|
28
|
-
List,
|
|
29
|
-
Optional,
|
|
30
|
-
Set,
|
|
31
|
-
Tuple,
|
|
32
|
-
cast,
|
|
33
|
-
get_args,
|
|
34
|
-
)
|
|
23
|
+
import re
|
|
24
|
+
from collections import OrderedDict
|
|
25
|
+
from datetime import datetime, timedelta
|
|
26
|
+
from typing import TYPE_CHECKING, Annotated, Any, Optional, Union, cast
|
|
35
27
|
from urllib.parse import quote_plus, unquote_plus
|
|
36
28
|
|
|
37
29
|
import geojson
|
|
@@ -39,235 +31,329 @@ import orjson
|
|
|
39
31
|
from dateutil.parser import isoparse
|
|
40
32
|
from dateutil.tz import tzutc
|
|
41
33
|
from jsonpath_ng import Child, Fields, Root
|
|
42
|
-
from pydantic import
|
|
34
|
+
from pydantic import Field
|
|
43
35
|
from pydantic.fields import FieldInfo
|
|
36
|
+
from requests.auth import AuthBase
|
|
37
|
+
from shapely.geometry.base import BaseGeometry
|
|
38
|
+
from typing_extensions import get_args
|
|
44
39
|
|
|
45
40
|
from eodag.api.product import EOProduct
|
|
46
41
|
from eodag.api.product.metadata_mapping import (
|
|
47
42
|
NOT_AVAILABLE,
|
|
48
43
|
NOT_MAPPED,
|
|
49
|
-
|
|
44
|
+
format_metadata,
|
|
45
|
+
format_query_params,
|
|
50
46
|
mtd_cfg_as_conversion_and_querypath,
|
|
51
47
|
properties_from_json,
|
|
52
48
|
)
|
|
53
49
|
from eodag.api.search_result import RawSearchResult
|
|
54
50
|
from eodag.plugins.search import PreparedSearch
|
|
55
|
-
from eodag.plugins.search.
|
|
56
|
-
from eodag.
|
|
57
|
-
from eodag.types import
|
|
58
|
-
from eodag.types.queryables import CommonQueryables
|
|
51
|
+
from eodag.plugins.search.qssearch import PostJsonSearch, QueryStringSearch
|
|
52
|
+
from eodag.types import json_field_definition_to_python
|
|
53
|
+
from eodag.types.queryables import Queryables, QueryablesDict
|
|
59
54
|
from eodag.utils import (
|
|
60
|
-
|
|
55
|
+
DEFAULT_SEARCH_TIMEOUT,
|
|
61
56
|
deepcopy,
|
|
62
57
|
dict_items_recursive_sort,
|
|
63
58
|
get_geometry_from_various,
|
|
64
|
-
|
|
65
|
-
from eodag.utils.constraints import (
|
|
66
|
-
fetch_constraints,
|
|
67
|
-
get_constraint_queryables_with_additional_params,
|
|
59
|
+
is_range_in_range,
|
|
68
60
|
)
|
|
69
61
|
from eodag.utils.exceptions import ValidationError
|
|
62
|
+
from eodag.utils.requests import fetch_json
|
|
70
63
|
|
|
71
64
|
if TYPE_CHECKING:
|
|
72
65
|
from eodag.config import PluginConfig
|
|
73
66
|
|
|
74
67
|
logger = logging.getLogger("eodag.search.build_search_result")
|
|
75
68
|
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
|
|
80
|
-
|
|
81
|
-
|
|
82
|
-
|
|
83
|
-
|
|
84
|
-
|
|
85
|
-
|
|
86
|
-
|
|
87
|
-
|
|
88
|
-
|
|
89
|
-
|
|
90
|
-
|
|
91
|
-
|
|
92
|
-
|
|
93
|
-
|
|
69
|
+
# keywords from ECMWF keyword database + "dataset" (not part of database but exists)
|
|
70
|
+
# database: https://confluence.ecmwf.int/display/UDOC/Keywords+in+MARS+and+Dissemination+requests
|
|
71
|
+
ECMWF_KEYWORDS = [
|
|
72
|
+
"dataset",
|
|
73
|
+
"accuracy",
|
|
74
|
+
"activity",
|
|
75
|
+
"anoffset",
|
|
76
|
+
"bitmap",
|
|
77
|
+
"block",
|
|
78
|
+
"channel",
|
|
79
|
+
"class",
|
|
80
|
+
"database",
|
|
81
|
+
"date",
|
|
82
|
+
"diagnostic",
|
|
83
|
+
"direction",
|
|
84
|
+
"domain",
|
|
85
|
+
"duplicates",
|
|
86
|
+
"expect",
|
|
87
|
+
"expver",
|
|
88
|
+
"fcmonth",
|
|
89
|
+
"fcperiod",
|
|
90
|
+
"fieldset",
|
|
91
|
+
"filter",
|
|
92
|
+
"format",
|
|
93
|
+
"frame",
|
|
94
|
+
"frequency",
|
|
95
|
+
"generation",
|
|
96
|
+
"grid",
|
|
97
|
+
"hdate",
|
|
98
|
+
"ident",
|
|
99
|
+
"interpolation",
|
|
100
|
+
"intgrid",
|
|
101
|
+
"iteration",
|
|
102
|
+
"latitude",
|
|
103
|
+
"levelist",
|
|
104
|
+
"levtype",
|
|
105
|
+
"longitude",
|
|
106
|
+
"lsm",
|
|
107
|
+
"method",
|
|
108
|
+
"number",
|
|
109
|
+
"obsgroup",
|
|
110
|
+
"obstype",
|
|
111
|
+
"origin",
|
|
112
|
+
"packing",
|
|
113
|
+
"padding",
|
|
114
|
+
"param",
|
|
115
|
+
"priority",
|
|
116
|
+
"product",
|
|
117
|
+
"range",
|
|
118
|
+
"realization",
|
|
119
|
+
"refdate",
|
|
120
|
+
"reference",
|
|
121
|
+
"reportype",
|
|
122
|
+
"repres",
|
|
123
|
+
"resolution",
|
|
124
|
+
"rotation",
|
|
125
|
+
"section",
|
|
126
|
+
"source",
|
|
127
|
+
"step",
|
|
128
|
+
"stream",
|
|
129
|
+
"system",
|
|
130
|
+
"target",
|
|
131
|
+
"time",
|
|
132
|
+
"truncation",
|
|
133
|
+
"type",
|
|
134
|
+
"use",
|
|
135
|
+
]
|
|
136
|
+
|
|
137
|
+
# additional keywords from copernicus services
|
|
138
|
+
COP_DS_KEYWORDS = [
|
|
139
|
+
"aerosol_type",
|
|
140
|
+
"altitude",
|
|
141
|
+
"product_type",
|
|
142
|
+
"band",
|
|
143
|
+
"cdr_type",
|
|
144
|
+
"data_format",
|
|
145
|
+
"dataset_type",
|
|
146
|
+
"day",
|
|
147
|
+
"download_format",
|
|
148
|
+
"ensemble_member",
|
|
149
|
+
"experiment",
|
|
150
|
+
"forcing_type",
|
|
151
|
+
"gcm",
|
|
152
|
+
"hday",
|
|
153
|
+
"hmonth",
|
|
154
|
+
"horizontal_resolution",
|
|
155
|
+
"hydrological_model",
|
|
156
|
+
"hydrological_year",
|
|
157
|
+
"hyear",
|
|
158
|
+
"input_observations",
|
|
159
|
+
"leadtime_hour",
|
|
160
|
+
"leadtime_month",
|
|
161
|
+
"level",
|
|
162
|
+
"location",
|
|
163
|
+
"model",
|
|
164
|
+
"model_level",
|
|
165
|
+
"model_levels",
|
|
166
|
+
"month",
|
|
167
|
+
"nominal_day",
|
|
168
|
+
"originating_centre",
|
|
169
|
+
"period",
|
|
170
|
+
"pressure_level",
|
|
171
|
+
"processing_level",
|
|
172
|
+
"processing_type",
|
|
173
|
+
"product_version",
|
|
174
|
+
"quantity",
|
|
175
|
+
"rcm",
|
|
176
|
+
"region",
|
|
177
|
+
"release_version",
|
|
178
|
+
"satellite",
|
|
179
|
+
"sensor",
|
|
180
|
+
"sensor_and_algorithm",
|
|
181
|
+
"soil_level",
|
|
182
|
+
"sky_type",
|
|
183
|
+
"statistic",
|
|
184
|
+
"system_version",
|
|
185
|
+
"temporal_aggregation",
|
|
186
|
+
"time_aggregation",
|
|
187
|
+
"time_reference",
|
|
188
|
+
"time_step",
|
|
189
|
+
"variable",
|
|
190
|
+
"variable_type",
|
|
191
|
+
"version",
|
|
192
|
+
"year",
|
|
193
|
+
]
|
|
194
|
+
|
|
195
|
+
|
|
196
|
+
def keywords_to_mdt(
|
|
197
|
+
keywords: list[str], prefix: Optional[str] = None
|
|
198
|
+
) -> dict[str, Any]:
|
|
94
199
|
"""
|
|
200
|
+
Make metadata mapping dict from a list of keywords
|
|
95
201
|
|
|
96
|
-
|
|
97
|
-
|
|
98
|
-
|
|
99
|
-
"""Count method that will always return 1."""
|
|
100
|
-
return 1
|
|
101
|
-
|
|
102
|
-
def collect_search_urls(
|
|
103
|
-
self,
|
|
104
|
-
prep: PreparedSearch = PreparedSearch(),
|
|
105
|
-
**kwargs: Any,
|
|
106
|
-
) -> Tuple[List[str], int]:
|
|
107
|
-
"""Wraps PostJsonSearch.collect_search_urls to force product count to 1"""
|
|
108
|
-
urls, _ = super(BuildPostSearchResult, self).collect_search_urls(prep, **kwargs)
|
|
109
|
-
return urls, 1
|
|
110
|
-
|
|
111
|
-
def do_search(
|
|
112
|
-
self, prep: PreparedSearch = PreparedSearch(items_per_page=None), **kwargs: Any
|
|
113
|
-
) -> List[Dict[str, Any]]:
|
|
114
|
-
"""Perform the actual search request, and return result in a single element."""
|
|
115
|
-
prep.url = prep.search_urls[0]
|
|
116
|
-
prep.info_message = f"Sending search request: {prep.url}"
|
|
117
|
-
prep.exception_message = (
|
|
118
|
-
f"Skipping error while searching for {self.provider}"
|
|
119
|
-
f" {self.__class__.__name__} instance"
|
|
120
|
-
)
|
|
121
|
-
response = self._request(prep)
|
|
122
|
-
|
|
123
|
-
return [response.json()]
|
|
124
|
-
|
|
125
|
-
def normalize_results(
|
|
126
|
-
self, results: RawSearchResult, **kwargs: Any
|
|
127
|
-
) -> List[EOProduct]:
|
|
128
|
-
"""Build :class:`~eodag.api.product._product.EOProduct` from provider result
|
|
129
|
-
|
|
130
|
-
:param results: Raw provider result as single dict in list
|
|
131
|
-
:param kwargs: Search arguments
|
|
132
|
-
:returns: list of single :class:`~eodag.api.product._product.EOProduct`
|
|
133
|
-
"""
|
|
134
|
-
product_type = kwargs.get("productType")
|
|
202
|
+
prefix:keyword:
|
|
203
|
+
- keyword
|
|
204
|
+
- $."prefix:keyword"
|
|
135
205
|
|
|
136
|
-
|
|
206
|
+
>>> keywords_to_mdt(["month", "year"])
|
|
207
|
+
{'month': ['month', '$."month"'], 'year': ['year', '$."year"']}
|
|
208
|
+
>>> keywords_to_mdt(["month", "year"], "ecmwf")
|
|
209
|
+
{'ecmwf:month': ['month', '$."ecmwf:month"'], 'ecmwf:year': ['year', '$."ecmwf:year"']}
|
|
137
210
|
|
|
138
|
-
|
|
139
|
-
|
|
140
|
-
|
|
141
|
-
|
|
142
|
-
|
|
143
|
-
|
|
144
|
-
|
|
145
|
-
|
|
146
|
-
|
|
147
|
-
|
|
148
|
-
|
|
149
|
-
|
|
150
|
-
|
|
151
|
-
|
|
152
|
-
|
|
153
|
-
|
|
154
|
-
|
|
155
|
-
|
|
156
|
-
|
|
157
|
-
|
|
158
|
-
|
|
159
|
-
|
|
160
|
-
|
|
161
|
-
|
|
162
|
-
|
|
163
|
-
|
|
164
|
-
|
|
165
|
-
|
|
166
|
-
|
|
167
|
-
|
|
168
|
-
|
|
169
|
-
|
|
170
|
-
|
|
171
|
-
|
|
172
|
-
|
|
173
|
-
|
|
174
|
-
|
|
175
|
-
|
|
176
|
-
|
|
177
|
-
|
|
178
|
-
|
|
179
|
-
|
|
180
|
-
|
|
181
|
-
|
|
182
|
-
|
|
183
|
-
|
|
184
|
-
|
|
185
|
-
|
|
186
|
-
|
|
187
|
-
|
|
188
|
-
|
|
189
|
-
|
|
190
|
-
|
|
191
|
-
|
|
192
|
-
|
|
193
|
-
|
|
194
|
-
|
|
195
|
-
|
|
196
|
-
|
|
197
|
-
|
|
198
|
-
|
|
199
|
-
|
|
200
|
-
|
|
201
|
-
|
|
202
|
-
|
|
203
|
-
|
|
204
|
-
|
|
205
|
-
|
|
206
|
-
|
|
211
|
+
:param keywords: List of keywords to be converted
|
|
212
|
+
:param prefix: prefix to be added to the parameter in the mapping
|
|
213
|
+
:return: metadata mapping dict
|
|
214
|
+
"""
|
|
215
|
+
mdt: dict[str, Any] = {}
|
|
216
|
+
for keyword in keywords:
|
|
217
|
+
key = f"{prefix}:{keyword}" if prefix else keyword
|
|
218
|
+
mdt[key] = [keyword, f'$."{key}"']
|
|
219
|
+
return mdt
|
|
220
|
+
|
|
221
|
+
|
|
222
|
+
def strip_quotes(value: Any) -> Any:
|
|
223
|
+
"""Strip superfluous quotes from elements (added by mapping converter to_geojson).
|
|
224
|
+
|
|
225
|
+
>>> strip_quotes("'abc'")
|
|
226
|
+
'abc'
|
|
227
|
+
>>> strip_quotes(["'abc'", '"def'])
|
|
228
|
+
['abc', 'def']
|
|
229
|
+
>>> strip_quotes({"'abc'": 'def"'})
|
|
230
|
+
{'abc': 'def'}
|
|
231
|
+
|
|
232
|
+
:param value: value from which quotes should be removed (should be either str or list)
|
|
233
|
+
:return: value without quotes
|
|
234
|
+
:raises: NotImplementedError
|
|
235
|
+
"""
|
|
236
|
+
if isinstance(value, (list, tuple)):
|
|
237
|
+
return [strip_quotes(v) for v in value]
|
|
238
|
+
elif isinstance(value, dict):
|
|
239
|
+
return {strip_quotes(k): strip_quotes(v) for k, v in value.items()}
|
|
240
|
+
else:
|
|
241
|
+
return str(value).strip("'\"")
|
|
242
|
+
|
|
243
|
+
|
|
244
|
+
def _update_properties_from_element(
|
|
245
|
+
prop: dict[str, Any], element: dict[str, Any], values: list[str]
|
|
246
|
+
) -> None:
|
|
247
|
+
"""updates a property dict with the given values based on the information from the element dict
|
|
248
|
+
e.g. the type is set based on the type of the element
|
|
249
|
+
"""
|
|
250
|
+
# multichoice elements are transformed into array
|
|
251
|
+
if element["type"] in ("StringListWidget", "StringListArrayWidget"):
|
|
252
|
+
prop["type"] = "array"
|
|
253
|
+
if values:
|
|
254
|
+
prop["items"] = {"type": "string", "enum": sorted(values)}
|
|
255
|
+
|
|
256
|
+
# single choice elements are transformed into string
|
|
257
|
+
elif element["type"] in (
|
|
258
|
+
"StringChoiceWidget",
|
|
259
|
+
"DateRangeWidget",
|
|
260
|
+
"FreeformInputWidget",
|
|
261
|
+
):
|
|
262
|
+
prop["type"] = "string"
|
|
263
|
+
if values:
|
|
264
|
+
prop["enum"] = sorted(values)
|
|
265
|
+
|
|
266
|
+
# a bbox element
|
|
267
|
+
elif element["type"] in ["GeographicExtentWidget", "GeographicExtentMapWidget"]:
|
|
268
|
+
prop.update(
|
|
269
|
+
{
|
|
270
|
+
"type": "array",
|
|
271
|
+
"minItems": 4,
|
|
272
|
+
"additionalItems": False,
|
|
273
|
+
"items": [
|
|
274
|
+
{
|
|
275
|
+
"type": "number",
|
|
276
|
+
"maximum": 180,
|
|
277
|
+
"minimum": -180,
|
|
278
|
+
"description": "West border of the bounding box",
|
|
279
|
+
},
|
|
280
|
+
{
|
|
281
|
+
"type": "number",
|
|
282
|
+
"maximum": 90,
|
|
283
|
+
"minimum": -90,
|
|
284
|
+
"description": "South border of the bounding box",
|
|
285
|
+
},
|
|
286
|
+
{
|
|
287
|
+
"type": "number",
|
|
288
|
+
"maximum": 180,
|
|
289
|
+
"minimum": -180,
|
|
290
|
+
"description": "East border of the bounding box",
|
|
291
|
+
},
|
|
292
|
+
{
|
|
293
|
+
"type": "number",
|
|
294
|
+
"maximum": 90,
|
|
295
|
+
"minimum": -90,
|
|
296
|
+
"description": "North border of the bounding box",
|
|
297
|
+
},
|
|
298
|
+
],
|
|
299
|
+
}
|
|
207
300
|
)
|
|
208
|
-
parsed_properties["id"] = parsed_properties["title"] = product_id
|
|
209
|
-
|
|
210
|
-
# update downloadLink and orderLink
|
|
211
|
-
parsed_properties["_dc_qs"] = quote_plus(qs)
|
|
212
|
-
if parsed_properties["downloadLink"] != "Not Available":
|
|
213
|
-
parsed_properties["downloadLink"] += f"?{qs}"
|
|
214
301
|
|
|
215
|
-
|
|
216
|
-
|
|
217
|
-
|
|
218
|
-
for param, mapping in self.config.metadata_mapping.items():
|
|
219
|
-
if dl_path in mapping or dl_path_from_root in mapping:
|
|
220
|
-
parsed_properties.update(
|
|
221
|
-
properties_from_json(parsed_properties, {param: mapping})
|
|
222
|
-
)
|
|
302
|
+
# DateRangeWidget is a calendar date picker
|
|
303
|
+
if element["type"] == "DateRangeWidget":
|
|
304
|
+
prop["description"] = "date formatted like yyyy-mm-dd/yyyy-mm-dd"
|
|
223
305
|
|
|
224
|
-
|
|
225
|
-
|
|
226
|
-
getattr(self.config, "product_type_config", {}),
|
|
227
|
-
**parsed_properties,
|
|
228
|
-
)
|
|
306
|
+
if description := element.get("help"):
|
|
307
|
+
prop["description"] = description
|
|
229
308
|
|
|
230
|
-
product = EOProduct(
|
|
231
|
-
provider=self.provider,
|
|
232
|
-
productType=product_type,
|
|
233
|
-
properties=parsed_properties,
|
|
234
|
-
)
|
|
235
309
|
|
|
236
|
-
|
|
237
|
-
|
|
238
|
-
|
|
310
|
+
def ecmwf_format(v: str) -> str:
|
|
311
|
+
"""Add ECMWF prefix to value v if v is a ECMWF keyword."""
|
|
312
|
+
return "ecmwf:" + v if v in ECMWF_KEYWORDS + COP_DS_KEYWORDS else v
|
|
239
313
|
|
|
240
314
|
|
|
241
|
-
class
|
|
242
|
-
"""
|
|
315
|
+
class ECMWFSearch(PostJsonSearch):
|
|
316
|
+
"""ECMWF search plugin.
|
|
243
317
|
|
|
244
|
-
This plugin builds a
|
|
318
|
+
This plugin builds a :class:`~eodag.api.search_result.SearchResult` containing a single product
|
|
245
319
|
using given query parameters as product properties.
|
|
246
320
|
|
|
247
|
-
The available configuration parameters inherits from parent classes
|
|
248
|
-
|
|
249
|
-
:class:`~eodag.plugins.search.qssearch.PostJsonSearch` and
|
|
250
|
-
:class:`~eodag.plugins.search.qssearch.QueryStringSearch`), with particularly for this plugin:
|
|
321
|
+
The available configuration parameters inherits from parent classes, with some particular parameters
|
|
322
|
+
for this plugin.
|
|
251
323
|
|
|
252
|
-
:param provider:
|
|
324
|
+
:param provider: An eodag providers configuration dictionary
|
|
253
325
|
:param config: Search plugin configuration:
|
|
254
326
|
|
|
255
|
-
* :attr:`~eodag.config.PluginConfig.
|
|
256
|
-
|
|
257
|
-
|
|
258
|
-
|
|
327
|
+
* :attr:`~eodag.config.PluginConfig.remove_from_query` (``list[str]``): List of parameters
|
|
328
|
+
used to parse metadata but that must not be included to the query
|
|
329
|
+
* :attr:`~eodag.config.PluginConfig.end_date_excluded` (``bool``): Set to `False` if
|
|
330
|
+
provider does not include end date to search
|
|
331
|
+
* :attr:`~eodag.config.PluginConfig.discover_queryables`
|
|
332
|
+
(:class:`~eodag.config.PluginConfig.DiscoverQueryables`): configuration to fetch the queryables from a
|
|
333
|
+
provider queryables endpoint; It has the following keys:
|
|
334
|
+
|
|
335
|
+
* :attr:`~eodag.config.PluginConfig.DiscoverQueryables.fetch_url` (``str``): url to fetch the queryables valid
|
|
336
|
+
for all product types
|
|
337
|
+
* :attr:`~eodag.config.PluginConfig.DiscoverQueryables.product_type_fetch_url` (``str``): url to fetch the
|
|
338
|
+
queryables for a specific product type
|
|
339
|
+
* :attr:`~eodag.config.PluginConfig.DiscoverQueryables.constraints_url` (``str``): url of the constraint file
|
|
340
|
+
used to build queryables
|
|
259
341
|
"""
|
|
260
342
|
|
|
261
343
|
def __init__(self, provider: str, config: PluginConfig) -> None:
|
|
262
|
-
|
|
263
|
-
|
|
344
|
+
config.metadata_mapping = {
|
|
345
|
+
**keywords_to_mdt(ECMWF_KEYWORDS + COP_DS_KEYWORDS, "ecmwf"),
|
|
346
|
+
**config.metadata_mapping,
|
|
347
|
+
}
|
|
348
|
+
|
|
349
|
+
super().__init__(provider, config)
|
|
264
350
|
|
|
265
351
|
self.config.__dict__.setdefault("api_endpoint", "")
|
|
266
352
|
|
|
267
353
|
# needed by QueryStringSearch.build_query_string / format_free_text_search
|
|
268
354
|
self.config.__dict__.setdefault("free_text_search_operations", {})
|
|
269
355
|
# needed for compatibility
|
|
270
|
-
self.config.
|
|
356
|
+
self.config.pagination.setdefault("next_page_query_obj", "{{}}")
|
|
271
357
|
|
|
272
358
|
# parse jsonpath on init: product type specific metadata-mapping
|
|
273
359
|
for product_type in self.config.products.keys():
|
|
@@ -304,46 +390,70 @@ class BuildSearchResult(BuildPostSearchResult):
|
|
|
304
390
|
"metadata_mapping"
|
|
305
391
|
] = product_type_metadata_mapping
|
|
306
392
|
|
|
307
|
-
def do_search(self, *args: Any, **kwargs: Any) ->
|
|
308
|
-
"""Should perform the actual search request.
|
|
393
|
+
def do_search(self, *args: Any, **kwargs: Any) -> list[dict[str, Any]]:
|
|
394
|
+
"""Should perform the actual search request.
|
|
395
|
+
|
|
396
|
+
:param args: arguments to be used in the search
|
|
397
|
+
:param kwargs: keyword arguments to be used in the search
|
|
398
|
+
:return: list containing the results from the provider in json format
|
|
399
|
+
"""
|
|
400
|
+
# no real search. We fake it all
|
|
309
401
|
return [{}]
|
|
310
402
|
|
|
311
403
|
def query(
|
|
312
404
|
self,
|
|
313
405
|
prep: PreparedSearch = PreparedSearch(),
|
|
314
406
|
**kwargs: Any,
|
|
315
|
-
) ->
|
|
316
|
-
"""Build ready-to-download SearchResult
|
|
407
|
+
) -> tuple[list[EOProduct], Optional[int]]:
|
|
408
|
+
"""Build ready-to-download SearchResult
|
|
317
409
|
|
|
318
|
-
|
|
410
|
+
:param prep: :class:`~eodag.plugins.search.PreparedSearch` object containing information needed for the search
|
|
411
|
+
:param kwargs: keyword arguments to be used in the search
|
|
412
|
+
:returns: list of products and number of products (optional)
|
|
413
|
+
"""
|
|
414
|
+
product_type = prep.product_type
|
|
415
|
+
if not product_type:
|
|
416
|
+
product_type = kwargs.get("productType", None)
|
|
417
|
+
self._preprocess_search_params(kwargs, product_type)
|
|
418
|
+
result, num_items = super().query(prep, **kwargs)
|
|
419
|
+
if prep.count and not num_items:
|
|
420
|
+
num_items = 1
|
|
319
421
|
|
|
320
|
-
return
|
|
422
|
+
return result, num_items
|
|
321
423
|
|
|
322
424
|
def clear(self) -> None:
|
|
323
425
|
"""Clear search context"""
|
|
324
|
-
|
|
426
|
+
super().clear()
|
|
325
427
|
|
|
326
428
|
def build_query_string(
|
|
327
429
|
self, product_type: str, **kwargs: Any
|
|
328
|
-
) ->
|
|
329
|
-
"""Build The query string using the search parameters
|
|
430
|
+
) -> tuple[dict[str, Any], str]:
|
|
431
|
+
"""Build The query string using the search parameters
|
|
432
|
+
|
|
433
|
+
:param product_type: product type id
|
|
434
|
+
:param kwargs: keyword arguments to be used in the query string
|
|
435
|
+
:return: formatted query params and encode query string
|
|
436
|
+
"""
|
|
330
437
|
# parse kwargs as properties as they might be needed to build the query
|
|
331
438
|
parsed_properties = properties_from_json(
|
|
332
439
|
kwargs,
|
|
333
440
|
self.config.metadata_mapping,
|
|
334
441
|
)
|
|
335
442
|
available_properties = {
|
|
336
|
-
|
|
443
|
+
# We strip values of superfluous quotes (added by mapping converter to_geojson).
|
|
444
|
+
k: strip_quotes(v)
|
|
337
445
|
for k, v in parsed_properties.items()
|
|
338
446
|
if v not in [NOT_AVAILABLE, NOT_MAPPED]
|
|
339
447
|
}
|
|
340
448
|
|
|
341
449
|
# build and return the query
|
|
342
|
-
return
|
|
343
|
-
|
|
450
|
+
return super().build_query_string(
|
|
451
|
+
product_type=product_type, **available_properties
|
|
344
452
|
)
|
|
345
453
|
|
|
346
|
-
def _preprocess_search_params(
|
|
454
|
+
def _preprocess_search_params(
|
|
455
|
+
self, params: dict[str, Any], product_type: Optional[str]
|
|
456
|
+
) -> None:
|
|
347
457
|
"""Preprocess search parameters before making a request to the CDS API.
|
|
348
458
|
|
|
349
459
|
This method is responsible for checking and updating the provided search parameters
|
|
@@ -352,6 +462,7 @@ class BuildSearchResult(BuildPostSearchResult):
|
|
|
352
462
|
in the input parameters, default values or values from the configuration are used.
|
|
353
463
|
|
|
354
464
|
:param params: Search parameters to be preprocessed.
|
|
465
|
+
:param product_type: (optional) product type id
|
|
355
466
|
"""
|
|
356
467
|
_dc_qs = params.get("_dc_qs", None)
|
|
357
468
|
if _dc_qs is not None:
|
|
@@ -378,165 +489,792 @@ class BuildSearchResult(BuildPostSearchResult):
|
|
|
378
489
|
non_none_params = {k: v for k, v in params.items() if v}
|
|
379
490
|
|
|
380
491
|
# productType
|
|
381
|
-
dataset = params.get("dataset", None)
|
|
492
|
+
dataset = params.get("ecmwf:dataset", None)
|
|
382
493
|
params["productType"] = non_none_params.get("productType", dataset)
|
|
383
494
|
|
|
384
495
|
# dates
|
|
385
|
-
|
|
386
|
-
|
|
387
|
-
|
|
388
|
-
).replace(
|
|
389
|
-
"Z", "+00:00"
|
|
390
|
-
) # before 3.11
|
|
391
|
-
)
|
|
392
|
-
|
|
393
|
-
default_end_from_cfg = self.config.products.get(params["productType"], {}).get(
|
|
394
|
-
"_default_end_date", None
|
|
395
|
-
)
|
|
396
|
-
default_end_str = (
|
|
397
|
-
default_end_from_cfg
|
|
398
|
-
or (
|
|
399
|
-
datetime.now(timezone.utc)
|
|
400
|
-
if params.get("startTimeFromAscendingNode")
|
|
401
|
-
else mission_start_dt + timedelta(days=1)
|
|
402
|
-
).isoformat()
|
|
403
|
-
)
|
|
404
|
-
|
|
405
|
-
params["startTimeFromAscendingNode"] = non_none_params.get(
|
|
406
|
-
"startTimeFromAscendingNode", mission_start_dt.isoformat()
|
|
407
|
-
)
|
|
408
|
-
params["completionTimeFromAscendingNode"] = non_none_params.get(
|
|
409
|
-
"completionTimeFromAscendingNode", default_end_str
|
|
410
|
-
)
|
|
496
|
+
# check if default dates have to be added
|
|
497
|
+
if getattr(self.config, "dates_required", False):
|
|
498
|
+
self._check_date_params(params, product_type)
|
|
411
499
|
|
|
412
500
|
# adapt end date if it is midnight
|
|
413
|
-
|
|
414
|
-
|
|
415
|
-
|
|
416
|
-
end_date = datetime.strptime(
|
|
417
|
-
params["completionTimeFromAscendingNode"], "%Y-%m-%dT%H:%M:%SZ"
|
|
418
|
-
)
|
|
419
|
-
end_date = end_date.replace(tzinfo=tzutc())
|
|
420
|
-
except ValueError:
|
|
501
|
+
if "completionTimeFromAscendingNode" in params:
|
|
502
|
+
end_date_excluded = getattr(self.config, "end_date_excluded", True)
|
|
503
|
+
is_datetime = True
|
|
421
504
|
try:
|
|
422
505
|
end_date = datetime.strptime(
|
|
423
|
-
params["completionTimeFromAscendingNode"], "%Y-%m-%dT%H:%M:%
|
|
506
|
+
params["completionTimeFromAscendingNode"], "%Y-%m-%dT%H:%M:%SZ"
|
|
424
507
|
)
|
|
425
508
|
end_date = end_date.replace(tzinfo=tzutc())
|
|
426
509
|
except ValueError:
|
|
427
|
-
|
|
428
|
-
|
|
429
|
-
|
|
430
|
-
|
|
431
|
-
|
|
432
|
-
|
|
433
|
-
|
|
434
|
-
|
|
435
|
-
|
|
436
|
-
|
|
437
|
-
|
|
510
|
+
try:
|
|
511
|
+
end_date = datetime.strptime(
|
|
512
|
+
params["completionTimeFromAscendingNode"],
|
|
513
|
+
"%Y-%m-%dT%H:%M:%S.%fZ",
|
|
514
|
+
)
|
|
515
|
+
end_date = end_date.replace(tzinfo=tzutc())
|
|
516
|
+
except ValueError:
|
|
517
|
+
end_date = isoparse(params["completionTimeFromAscendingNode"])
|
|
518
|
+
is_datetime = False
|
|
519
|
+
start_date = isoparse(params["startTimeFromAscendingNode"])
|
|
520
|
+
if (
|
|
521
|
+
not end_date_excluded
|
|
522
|
+
and is_datetime
|
|
523
|
+
and end_date > start_date
|
|
524
|
+
and end_date
|
|
525
|
+
== end_date.replace(hour=0, minute=0, second=0, microsecond=0)
|
|
526
|
+
):
|
|
527
|
+
end_date += timedelta(days=-1)
|
|
528
|
+
params["completionTimeFromAscendingNode"] = end_date.isoformat()
|
|
438
529
|
|
|
439
530
|
# geometry
|
|
440
531
|
if "geometry" in params:
|
|
441
532
|
params["geometry"] = get_geometry_from_various(geometry=params["geometry"])
|
|
442
533
|
|
|
534
|
+
def _get_product_type_queryables(
|
|
535
|
+
self, product_type: Optional[str], alias: Optional[str], filters: dict[str, Any]
|
|
536
|
+
) -> QueryablesDict:
|
|
537
|
+
"""Override to set additional_properties to false."""
|
|
538
|
+
default_values: dict[str, Any] = deepcopy(
|
|
539
|
+
getattr(self.config, "products", {}).get(product_type, {})
|
|
540
|
+
)
|
|
541
|
+
default_values.pop("metadata_mapping", None)
|
|
542
|
+
|
|
543
|
+
filters["productType"] = product_type
|
|
544
|
+
queryables = self.discover_queryables(**{**default_values, **filters}) or {}
|
|
545
|
+
|
|
546
|
+
return QueryablesDict(additional_properties=False, **queryables)
|
|
547
|
+
|
|
443
548
|
def discover_queryables(
|
|
444
549
|
self, **kwargs: Any
|
|
445
|
-
) -> Optional[
|
|
550
|
+
) -> Optional[dict[str, Annotated[Any, FieldInfo]]]:
|
|
446
551
|
"""Fetch queryables list from provider using its constraints file
|
|
447
552
|
|
|
448
553
|
:param kwargs: additional filters for queryables (`productType` and other search
|
|
449
554
|
arguments)
|
|
450
555
|
:returns: fetched queryable parameters dict
|
|
451
556
|
"""
|
|
452
|
-
|
|
453
|
-
|
|
454
|
-
|
|
455
|
-
|
|
456
|
-
|
|
457
|
-
|
|
557
|
+
product_type = kwargs.pop("productType")
|
|
558
|
+
product_type_config = self.config.products.get(product_type, {})
|
|
559
|
+
provider_product_type = (
|
|
560
|
+
product_type_config.get("ecmwf:dataset", None)
|
|
561
|
+
or product_type_config["productType"]
|
|
562
|
+
)
|
|
563
|
+
if "start" in kwargs:
|
|
564
|
+
kwargs["startTimeFromAscendingNode"] = kwargs.pop("start")
|
|
565
|
+
if "end" in kwargs:
|
|
566
|
+
kwargs["completionTimeFromAscendingNode"] = kwargs.pop("end")
|
|
567
|
+
|
|
568
|
+
# extract default datetime
|
|
569
|
+
processed_kwargs = deepcopy(kwargs)
|
|
570
|
+
self._preprocess_search_params(processed_kwargs, product_type)
|
|
571
|
+
|
|
572
|
+
constraints_url = format_metadata(
|
|
573
|
+
getattr(self.config, "discover_queryables", {}).get("constraints_url", ""),
|
|
574
|
+
**kwargs,
|
|
575
|
+
)
|
|
576
|
+
constraints: list[dict[str, Any]] = self._fetch_data(constraints_url)
|
|
458
577
|
|
|
459
|
-
|
|
460
|
-
"
|
|
578
|
+
form_url = format_metadata(
|
|
579
|
+
getattr(self.config, "discover_queryables", {}).get("form_url", ""),
|
|
580
|
+
**kwargs,
|
|
461
581
|
)
|
|
462
|
-
|
|
582
|
+
form: list[dict[str, Any]] = self._fetch_data(form_url)
|
|
583
|
+
|
|
584
|
+
formated_kwargs = self.format_as_provider_keyword(
|
|
585
|
+
product_type, processed_kwargs
|
|
586
|
+
)
|
|
587
|
+
# we re-apply kwargs input to consider override of year, month, day and time.
|
|
588
|
+
for key in kwargs:
|
|
589
|
+
if key.startswith("ecmwf:"):
|
|
590
|
+
formated_kwargs[key.replace("ecmwf:", "")] = kwargs[key]
|
|
591
|
+
elif key in (
|
|
592
|
+
"startTimeFromAscendingNode",
|
|
593
|
+
"completionTimeFromAscendingNode",
|
|
594
|
+
"geom",
|
|
595
|
+
):
|
|
596
|
+
formated_kwargs[key] = kwargs[key]
|
|
597
|
+
else:
|
|
598
|
+
raise ValidationError(
|
|
599
|
+
f"{key} is not a queryable parameter for {self.provider}"
|
|
600
|
+
)
|
|
601
|
+
|
|
602
|
+
# we use non empty kwargs as default to integrate user inputs
|
|
603
|
+
# it is needed because pydantic json schema does not represent "value"
|
|
604
|
+
# but only "default"
|
|
605
|
+
non_empty_formated: dict[str, Any] = {
|
|
606
|
+
k: v
|
|
607
|
+
for k, v in formated_kwargs.items()
|
|
608
|
+
if v and (not isinstance(v, list) or all(v))
|
|
609
|
+
}
|
|
610
|
+
non_empty_kwargs: dict[str, Any] = {
|
|
611
|
+
k: v
|
|
612
|
+
for k, v in processed_kwargs.items()
|
|
613
|
+
if v and (not isinstance(v, list) or all(v))
|
|
614
|
+
}
|
|
615
|
+
|
|
616
|
+
required_keywords: set[str] = set()
|
|
617
|
+
|
|
618
|
+
# calculate available values
|
|
619
|
+
if constraints:
|
|
620
|
+
# Apply constraints filtering
|
|
621
|
+
available_values = self.available_values_from_constraints(
|
|
622
|
+
constraints,
|
|
623
|
+
non_empty_formated,
|
|
624
|
+
form_keywords=[f["name"] for f in form],
|
|
625
|
+
)
|
|
626
|
+
|
|
627
|
+
# Pre-compute the required keywords (present in all constraint dicts)
|
|
628
|
+
# when form, required keywords are extracted directly from form
|
|
629
|
+
if not form:
|
|
630
|
+
required_keywords = set(constraints[0].keys())
|
|
631
|
+
for constraint in constraints[1:]:
|
|
632
|
+
required_keywords.intersection_update(constraint.keys())
|
|
633
|
+
else:
|
|
634
|
+
values_url = getattr(self.config, "available_values_url", "")
|
|
635
|
+
if not values_url:
|
|
636
|
+
return self.queryables_from_metadata_mapping(product_type)
|
|
637
|
+
if "{" in values_url:
|
|
638
|
+
values_url = values_url.format(productType=provider_product_type)
|
|
639
|
+
data = self._fetch_data(values_url)
|
|
640
|
+
available_values = data["constraints"]
|
|
641
|
+
required_keywords = data.get("required", [])
|
|
642
|
+
|
|
643
|
+
# To check if all keywords are queryable parameters, we check if they are in the
|
|
644
|
+
# available values or the product type config (available values calculated from the
|
|
645
|
+
# constraints might not include all queryables)
|
|
646
|
+
for keyword in kwargs:
|
|
647
|
+
if (
|
|
648
|
+
keyword
|
|
649
|
+
not in available_values.keys()
|
|
650
|
+
| product_type_config.keys()
|
|
651
|
+
| {
|
|
652
|
+
"startTimeFromAscendingNode",
|
|
653
|
+
"completionTimeFromAscendingNode",
|
|
654
|
+
"geom",
|
|
655
|
+
}
|
|
656
|
+
and keyword not in [f["name"] for f in form]
|
|
657
|
+
and keyword.replace("ecmwf:", "")
|
|
658
|
+
not in set(list(available_values.keys()) + [f["name"] for f in form])
|
|
659
|
+
):
|
|
660
|
+
raise ValidationError(f"{keyword} is not a queryable parameter")
|
|
661
|
+
|
|
662
|
+
# generate queryables
|
|
663
|
+
if form:
|
|
664
|
+
queryables = self.queryables_by_form(
|
|
665
|
+
form,
|
|
666
|
+
available_values,
|
|
667
|
+
non_empty_formated,
|
|
668
|
+
)
|
|
669
|
+
else:
|
|
670
|
+
queryables = self.queryables_by_values(
|
|
671
|
+
available_values, list(required_keywords), non_empty_kwargs
|
|
672
|
+
)
|
|
673
|
+
|
|
674
|
+
# ecmwf:date is replaced by start and end.
|
|
675
|
+
# start and end filters are supported whenever combinations of "year", "month", "day" filters exist
|
|
463
676
|
if (
|
|
464
|
-
|
|
465
|
-
|
|
677
|
+
queryables.pop("ecmwf:date", None)
|
|
678
|
+
or "ecmwf:year" in queryables
|
|
679
|
+
or "ecmwf:hyear" in queryables
|
|
466
680
|
):
|
|
467
|
-
|
|
468
|
-
|
|
681
|
+
queryables.update(
|
|
682
|
+
{
|
|
683
|
+
"start": Queryables.get_with_default(
|
|
684
|
+
"start", non_empty_kwargs.get("startTimeFromAscendingNode")
|
|
685
|
+
),
|
|
686
|
+
"end": Queryables.get_with_default(
|
|
687
|
+
"end",
|
|
688
|
+
non_empty_kwargs.get("completionTimeFromAscendingNode"),
|
|
689
|
+
),
|
|
690
|
+
}
|
|
469
691
|
)
|
|
470
692
|
|
|
471
|
-
#
|
|
472
|
-
|
|
473
|
-
|
|
474
|
-
|
|
693
|
+
# area is geom in EODAG.
|
|
694
|
+
if queryables.pop("area", None):
|
|
695
|
+
queryables["geom"] = Annotated[
|
|
696
|
+
Union[str, dict[str, float], BaseGeometry],
|
|
697
|
+
Field(
|
|
698
|
+
None,
|
|
699
|
+
description="Read EODAG documentation for all supported geometry format.",
|
|
700
|
+
),
|
|
701
|
+
]
|
|
475
702
|
|
|
476
|
-
|
|
703
|
+
return queryables
|
|
477
704
|
|
|
478
|
-
|
|
479
|
-
|
|
480
|
-
|
|
481
|
-
|
|
482
|
-
|
|
483
|
-
|
|
484
|
-
|
|
485
|
-
|
|
486
|
-
|
|
487
|
-
|
|
488
|
-
|
|
489
|
-
|
|
490
|
-
|
|
491
|
-
|
|
492
|
-
|
|
493
|
-
|
|
494
|
-
|
|
495
|
-
|
|
496
|
-
|
|
497
|
-
|
|
498
|
-
|
|
499
|
-
|
|
500
|
-
|
|
705
|
+
def available_values_from_constraints(
|
|
706
|
+
self,
|
|
707
|
+
constraints: list[dict[str, Any]],
|
|
708
|
+
input_keywords: dict[str, Any],
|
|
709
|
+
form_keywords: list[str],
|
|
710
|
+
) -> dict[str, list[str]]:
|
|
711
|
+
"""
|
|
712
|
+
Filter constraints using input_keywords. Return list of available queryables.
|
|
713
|
+
All constraint entries must have the same parameters.
|
|
714
|
+
|
|
715
|
+
:param constraints: list of constraints received from the provider
|
|
716
|
+
:param input_keywords: dict of input parameters given by the user
|
|
717
|
+
:param form_keywords: list of keyword names from the provider form endpoint
|
|
718
|
+
:return: dict with available values for each parameter
|
|
719
|
+
"""
|
|
720
|
+
# get ordered constraint keywords
|
|
721
|
+
constraints_keywords = list(
|
|
722
|
+
OrderedDict.fromkeys(k for c in constraints for k in c.keys())
|
|
723
|
+
)
|
|
724
|
+
|
|
725
|
+
# prepare ordered input keywords formatted as provider's keywords
|
|
726
|
+
# required to filter with constraints
|
|
727
|
+
ordered_keywords = (
|
|
728
|
+
[kw for kw in form_keywords if kw in constraints_keywords]
|
|
729
|
+
if form_keywords
|
|
730
|
+
else constraints_keywords
|
|
731
|
+
)
|
|
732
|
+
|
|
733
|
+
# filter constraint entries matching input keyword values
|
|
734
|
+
filtered_constraints: list[dict[str, Any]]
|
|
735
|
+
|
|
736
|
+
parsed_keywords: list[str] = []
|
|
737
|
+
for keyword in ordered_keywords:
|
|
738
|
+
values = input_keywords.get(keyword)
|
|
739
|
+
|
|
740
|
+
if values is None:
|
|
741
|
+
parsed_keywords.append(keyword)
|
|
742
|
+
continue
|
|
743
|
+
|
|
744
|
+
# we only compare list of strings.
|
|
745
|
+
if isinstance(values, dict):
|
|
746
|
+
raise ValidationError(
|
|
747
|
+
f"Parameter value as object is not supported: {keyword}={values}"
|
|
748
|
+
)
|
|
749
|
+
filter_v = values if isinstance(values, (list, tuple)) else [values]
|
|
750
|
+
|
|
751
|
+
# We convert every single value to a list of string
|
|
752
|
+
# We strip values of superfluous quotes (added by mapping converter to_geojson).
|
|
753
|
+
# ECMWF accept values with /to/. We need to split it to an array
|
|
754
|
+
# ECMWF accept values in format val1/val2. We need to split it to an array
|
|
755
|
+
sep = re.compile(r"/to/|/")
|
|
756
|
+
filter_v = [i for v in filter_v for i in sep.split(strip_quotes(v))]
|
|
757
|
+
|
|
758
|
+
# special handling for time 0000 converted to 0 by pre-formating with metadata_mapping
|
|
759
|
+
if keyword.split(":")[-1] == "time":
|
|
760
|
+
filter_v = ["0000" if str(v) == "0" else v for v in filter_v]
|
|
761
|
+
|
|
762
|
+
# Collect missing values to report errors
|
|
763
|
+
missing_values = set(filter_v)
|
|
764
|
+
|
|
765
|
+
# Filter constraints and check for missing values
|
|
766
|
+
filtered_constraints = []
|
|
767
|
+
for entry in constraints:
|
|
768
|
+
# Filter based on the presence of any value in filter_v
|
|
769
|
+
entry_values = entry.get(keyword, [])
|
|
770
|
+
|
|
771
|
+
# date constraint may be intervals. We identify intervals with a "/" in the value
|
|
772
|
+
# we assume that if the first value is an interval, all values are intervals
|
|
773
|
+
present_values = []
|
|
774
|
+
if keyword == "date" and "/" in entry[keyword][0]:
|
|
775
|
+
if any(is_range_in_range(x, values[0]) for x in entry[keyword]):
|
|
776
|
+
present_values = filter_v
|
|
777
|
+
else:
|
|
778
|
+
present_values = [
|
|
779
|
+
value for value in filter_v if value in entry_values
|
|
780
|
+
]
|
|
781
|
+
|
|
782
|
+
# Remove present values from the missing_values set
|
|
783
|
+
missing_values -= set(present_values)
|
|
784
|
+
|
|
785
|
+
if present_values:
|
|
786
|
+
filtered_constraints.append(entry)
|
|
787
|
+
|
|
788
|
+
# raise an error as no constraint entry matched the input keywords
|
|
789
|
+
# raise an error if one value from input is not allowed
|
|
790
|
+
if not filtered_constraints or missing_values:
|
|
791
|
+
allowed_values = list(
|
|
792
|
+
{value for c in constraints for value in c.get(keyword, [])}
|
|
793
|
+
)
|
|
794
|
+
# restore ecmwf: prefix before raising error
|
|
795
|
+
keyword = f"ecmwf:{keyword}"
|
|
796
|
+
|
|
797
|
+
all_keywords_str = ""
|
|
798
|
+
if len(parsed_keywords) > 1:
|
|
799
|
+
keywords = [
|
|
800
|
+
f"ecmwf:{k}={pk}"
|
|
801
|
+
for k in parsed_keywords
|
|
802
|
+
if (pk := input_keywords.get(k))
|
|
803
|
+
]
|
|
804
|
+
all_keywords_str = f" with {', '.join(keywords)}"
|
|
805
|
+
|
|
806
|
+
raise ValidationError(
|
|
807
|
+
f"{keyword}={values} is not available"
|
|
808
|
+
f"{all_keywords_str}."
|
|
809
|
+
f" Allowed values are {', '.join(allowed_values)}."
|
|
810
|
+
)
|
|
811
|
+
|
|
812
|
+
parsed_keywords.append(keyword)
|
|
813
|
+
constraints = filtered_constraints
|
|
814
|
+
|
|
815
|
+
available_values: dict[str, Any] = {k: set() for k in ordered_keywords}
|
|
816
|
+
|
|
817
|
+
# we aggregate the constraint entries left
|
|
818
|
+
for entry in constraints:
|
|
819
|
+
for key, value in entry.items():
|
|
820
|
+
available_values[key].update(value)
|
|
821
|
+
|
|
822
|
+
return {k: list(v) for k, v in available_values.items()}
|
|
823
|
+
|
|
824
|
+
def queryables_by_form(
|
|
825
|
+
self,
|
|
826
|
+
form: list[dict[str, Any]],
|
|
827
|
+
available_values: dict[str, list[str]],
|
|
828
|
+
defaults: dict[str, Any],
|
|
829
|
+
) -> dict[str, Annotated[Any, FieldInfo]]:
|
|
830
|
+
"""
|
|
831
|
+
Generate Annotated field definitions from form entries and available values
|
|
832
|
+
Used by Copernicus services like cop_cds, cop_ads, cop_ewds.
|
|
833
|
+
|
|
834
|
+
:param form: data fetched from the form endpoint of the provider
|
|
835
|
+
:param available_values: available values for each parameter
|
|
836
|
+
:param defaults: default values for the parameters
|
|
837
|
+
:return: dict of annotated queryables
|
|
838
|
+
"""
|
|
839
|
+
queryables: dict[str, Annotated[Any, FieldInfo]] = {}
|
|
840
|
+
|
|
841
|
+
required_list: list[str] = []
|
|
842
|
+
for element in form:
|
|
843
|
+
name: str = element["name"]
|
|
844
|
+
|
|
845
|
+
# those are not parameter elements.
|
|
846
|
+
if name in ("area_group", "global", "warning", "licences"):
|
|
847
|
+
continue
|
|
848
|
+
if "type" not in element or element["type"] == "FreeEditionWidget":
|
|
849
|
+
continue
|
|
850
|
+
|
|
851
|
+
# ordering done by id -> set id to high value if not present -> element will be last
|
|
852
|
+
if "id" not in element:
|
|
853
|
+
element["id"] = 100
|
|
854
|
+
|
|
855
|
+
prop = {"title": element.get("label", name)}
|
|
856
|
+
|
|
857
|
+
details = element.get("details", {})
|
|
858
|
+
|
|
859
|
+
# add values from form if keyword was not in constraints
|
|
860
|
+
values = (
|
|
861
|
+
available_values[name]
|
|
862
|
+
if name in available_values
|
|
863
|
+
else details.get("values")
|
|
501
864
|
)
|
|
502
|
-
|
|
503
|
-
|
|
504
|
-
|
|
505
|
-
|
|
506
|
-
|
|
507
|
-
|
|
508
|
-
|
|
509
|
-
|
|
510
|
-
|
|
511
|
-
|
|
512
|
-
|
|
513
|
-
|
|
514
|
-
|
|
515
|
-
|
|
865
|
+
|
|
866
|
+
# updates the properties with the values given based on the information from the element
|
|
867
|
+
_update_properties_from_element(prop, element, values)
|
|
868
|
+
|
|
869
|
+
default = defaults.get(name)
|
|
870
|
+
|
|
871
|
+
if details:
|
|
872
|
+
fields = details.get("fields")
|
|
873
|
+
if fields and (comment := fields[0].get("comment")):
|
|
874
|
+
prop["description"] = comment
|
|
875
|
+
|
|
876
|
+
if name == "area" and isinstance(default, dict):
|
|
877
|
+
default = list(default.values())
|
|
878
|
+
|
|
879
|
+
if default:
|
|
880
|
+
# We strip values of superfluous quotes (addded by mapping converter to_geojson).
|
|
881
|
+
default = strip_quotes(default)
|
|
882
|
+
|
|
883
|
+
# sometimes form returns default as array instead of string
|
|
884
|
+
if default and prop["type"] == "string" and isinstance(default, list):
|
|
885
|
+
default = ",".join(default)
|
|
886
|
+
|
|
887
|
+
is_required = bool(element.get("required"))
|
|
888
|
+
if is_required:
|
|
889
|
+
required_list.append(name)
|
|
890
|
+
|
|
891
|
+
queryables[ecmwf_format(name)] = Annotated[
|
|
892
|
+
get_args(
|
|
893
|
+
json_field_definition_to_python(
|
|
894
|
+
prop,
|
|
895
|
+
default_value=default,
|
|
896
|
+
required=is_required,
|
|
516
897
|
)
|
|
517
|
-
|
|
518
|
-
|
|
519
|
-
|
|
520
|
-
|
|
521
|
-
|
|
522
|
-
|
|
898
|
+
)
|
|
899
|
+
]
|
|
900
|
+
|
|
901
|
+
return queryables
|
|
902
|
+
|
|
903
|
+
def queryables_by_values(
|
|
904
|
+
self,
|
|
905
|
+
available_values: dict[str, list[str]],
|
|
906
|
+
required_keywords: list[str],
|
|
907
|
+
defaults: dict[str, Any],
|
|
908
|
+
) -> dict[str, Annotated[Any, FieldInfo]]:
|
|
909
|
+
"""
|
|
910
|
+
Generate Annotated field definitions from available values.
|
|
911
|
+
Used by ECMWF data providers like dedt_lumi.
|
|
912
|
+
|
|
913
|
+
:param available_values: available values for each parameter
|
|
914
|
+
:param required_keywords: list of required parameters
|
|
915
|
+
:param defaults: default values for the parameters
|
|
916
|
+
:return: dict of annotated queryables
|
|
917
|
+
"""
|
|
918
|
+
# Rename keywords from form with metadata mapping.
|
|
919
|
+
# Needed to map constraints like "xxxx" to eodag parameter "ecmwf:xxxx"
|
|
920
|
+
required = [ecmwf_format(k) for k in required_keywords]
|
|
921
|
+
|
|
922
|
+
queryables: dict[str, Annotated[Any, FieldInfo]] = {}
|
|
923
|
+
for name, values in available_values.items():
|
|
924
|
+
# Rename keywords from form with metadata mapping.
|
|
925
|
+
# Needed to map constraints like "xxxx" to eodag parameter "ecmwf:xxxx"
|
|
926
|
+
key = ecmwf_format(name)
|
|
927
|
+
|
|
928
|
+
default = defaults.get(key)
|
|
929
|
+
|
|
930
|
+
queryables[key] = Annotated[
|
|
931
|
+
get_args(
|
|
932
|
+
json_field_definition_to_python(
|
|
933
|
+
{"type": "string", "title": name, "enum": values},
|
|
934
|
+
default_value=strip_quotes(default) if default else None,
|
|
935
|
+
required=bool(key in required),
|
|
523
936
|
)
|
|
937
|
+
)
|
|
938
|
+
]
|
|
939
|
+
|
|
940
|
+
return queryables
|
|
941
|
+
|
|
942
|
+
def format_as_provider_keyword(
|
|
943
|
+
self, product_type: str, properties: dict[str, Any]
|
|
944
|
+
) -> dict[str, Any]:
|
|
945
|
+
"""Return provider equivalent keyword names from EODAG keywords.
|
|
946
|
+
|
|
947
|
+
:param product_type: product type id
|
|
948
|
+
:param properties: dict of properties to be formatted
|
|
949
|
+
:return: dict of formatted properties
|
|
950
|
+
"""
|
|
951
|
+
parsed_properties = properties_from_json(
|
|
952
|
+
properties,
|
|
953
|
+
self.config.metadata_mapping,
|
|
954
|
+
)
|
|
955
|
+
available_properties = {
|
|
956
|
+
k: v
|
|
957
|
+
for k, v in parsed_properties.items()
|
|
958
|
+
if v not in [NOT_AVAILABLE, NOT_MAPPED]
|
|
959
|
+
}
|
|
960
|
+
return format_query_params(product_type, self.config, available_properties)
|
|
961
|
+
|
|
962
|
+
def _fetch_data(self, url: str) -> Any:
|
|
963
|
+
"""
|
|
964
|
+
fetches from a provider elements like constraints or forms.
|
|
965
|
+
|
|
966
|
+
:param url: url from which the constraints can be fetched
|
|
967
|
+
:returns: json file content fetched from the provider
|
|
968
|
+
"""
|
|
969
|
+
if not url:
|
|
970
|
+
return []
|
|
971
|
+
|
|
972
|
+
auth = (
|
|
973
|
+
self.auth
|
|
974
|
+
if hasattr(self, "auth") and isinstance(self.auth, AuthBase)
|
|
975
|
+
else None
|
|
976
|
+
)
|
|
977
|
+
timeout = getattr(self.config, "timeout", DEFAULT_SEARCH_TIMEOUT)
|
|
978
|
+
return functools.lru_cache()(fetch_json)(url, auth=auth, timeout=timeout)
|
|
979
|
+
|
|
980
|
+
def normalize_results(
|
|
981
|
+
self, results: RawSearchResult, **kwargs: Any
|
|
982
|
+
) -> list[EOProduct]:
|
|
983
|
+
"""Build :class:`~eodag.api.product._product.EOProduct` from provider result
|
|
984
|
+
|
|
985
|
+
:param results: Raw provider result as single dict in list
|
|
986
|
+
:param kwargs: Search arguments
|
|
987
|
+
:returns: list of single :class:`~eodag.api.product._product.EOProduct`
|
|
988
|
+
"""
|
|
524
989
|
|
|
525
|
-
|
|
526
|
-
|
|
527
|
-
|
|
528
|
-
|
|
529
|
-
|
|
990
|
+
product_type = kwargs.get("productType")
|
|
991
|
+
|
|
992
|
+
result = results[0]
|
|
993
|
+
|
|
994
|
+
# datacube query string got from previous search
|
|
995
|
+
_dc_qs = kwargs.pop("_dc_qs", None)
|
|
996
|
+
if _dc_qs is not None:
|
|
997
|
+
qs = unquote_plus(unquote_plus(_dc_qs))
|
|
998
|
+
sorted_unpaginated_query_params = geojson.loads(qs)
|
|
999
|
+
else:
|
|
1000
|
+
# update result with query parameters without pagination (or search-only params)
|
|
1001
|
+
if isinstance(
|
|
1002
|
+
self.config.pagination["next_page_query_obj"], str
|
|
1003
|
+
) and hasattr(results, "query_params_unpaginated"):
|
|
1004
|
+
unpaginated_query_params = results.query_params_unpaginated
|
|
1005
|
+
elif isinstance(self.config.pagination["next_page_query_obj"], str):
|
|
1006
|
+
next_page_query_obj = orjson.loads(
|
|
1007
|
+
self.config.pagination["next_page_query_obj"].format()
|
|
530
1008
|
)
|
|
531
|
-
|
|
1009
|
+
unpaginated_query_params = {
|
|
1010
|
+
k: v
|
|
1011
|
+
for k, v in results.query_params.items()
|
|
1012
|
+
if (k, v) not in next_page_query_obj.items()
|
|
1013
|
+
}
|
|
1014
|
+
else:
|
|
1015
|
+
unpaginated_query_params = self.query_params
|
|
1016
|
+
# query hash, will be used to build a product id
|
|
1017
|
+
sorted_unpaginated_query_params = dict_items_recursive_sort(
|
|
1018
|
+
unpaginated_query_params
|
|
532
1019
|
)
|
|
533
|
-
|
|
534
|
-
|
|
535
|
-
|
|
536
|
-
|
|
537
|
-
|
|
1020
|
+
|
|
1021
|
+
# use all available query_params to parse properties
|
|
1022
|
+
result = dict(
|
|
1023
|
+
result,
|
|
1024
|
+
**sorted_unpaginated_query_params,
|
|
1025
|
+
qs=sorted_unpaginated_query_params,
|
|
1026
|
+
)
|
|
1027
|
+
|
|
1028
|
+
# remove unwanted query params
|
|
1029
|
+
for param in getattr(self.config, "remove_from_query", []):
|
|
1030
|
+
sorted_unpaginated_query_params.pop(param, None)
|
|
1031
|
+
|
|
1032
|
+
qs = geojson.dumps(sorted_unpaginated_query_params)
|
|
1033
|
+
|
|
1034
|
+
query_hash = hashlib.sha1(str(qs).encode("UTF-8")).hexdigest()
|
|
1035
|
+
|
|
1036
|
+
# update result with product_type_def_params and search args if not None (and not auth)
|
|
1037
|
+
kwargs.pop("auth", None)
|
|
1038
|
+
result.update(results.product_type_def_params)
|
|
1039
|
+
result = dict(result, **{k: v for k, v in kwargs.items() if v is not None})
|
|
1040
|
+
|
|
1041
|
+
# parse properties
|
|
1042
|
+
parsed_properties = properties_from_json(
|
|
1043
|
+
result,
|
|
1044
|
+
self.config.metadata_mapping,
|
|
1045
|
+
discovery_config=getattr(self.config, "discover_metadata", {}),
|
|
1046
|
+
)
|
|
1047
|
+
|
|
1048
|
+
if not product_type:
|
|
1049
|
+
product_type = parsed_properties.get("productType", None)
|
|
1050
|
+
|
|
1051
|
+
# build product id
|
|
1052
|
+
id_prefix = (product_type or self.provider).upper()
|
|
1053
|
+
if (
|
|
1054
|
+
"startTimeFromAscendingNode" in parsed_properties
|
|
1055
|
+
and parsed_properties["startTimeFromAscendingNode"] != "Not Available"
|
|
1056
|
+
and "completionTimeFromAscendingNode" in parsed_properties
|
|
1057
|
+
and parsed_properties["completionTimeFromAscendingNode"] != "Not Available"
|
|
1058
|
+
):
|
|
1059
|
+
product_id = "%s_%s_%s_%s" % (
|
|
1060
|
+
id_prefix,
|
|
1061
|
+
parsed_properties["startTimeFromAscendingNode"]
|
|
1062
|
+
.split("T")[0]
|
|
1063
|
+
.replace("-", ""),
|
|
1064
|
+
parsed_properties["completionTimeFromAscendingNode"]
|
|
1065
|
+
.split("T")[0]
|
|
1066
|
+
.replace("-", ""),
|
|
1067
|
+
query_hash,
|
|
538
1068
|
)
|
|
539
|
-
|
|
1069
|
+
elif (
|
|
1070
|
+
"startTimeFromAscendingNode" in parsed_properties
|
|
1071
|
+
and parsed_properties["startTimeFromAscendingNode"] != "Not Available"
|
|
1072
|
+
):
|
|
1073
|
+
product_id = "%s_%s_%s" % (
|
|
1074
|
+
id_prefix,
|
|
1075
|
+
parsed_properties["startTimeFromAscendingNode"]
|
|
1076
|
+
.split("T")[0]
|
|
1077
|
+
.replace("-", ""),
|
|
1078
|
+
query_hash,
|
|
1079
|
+
)
|
|
1080
|
+
else:
|
|
1081
|
+
product_id = f"{id_prefix}_{query_hash}"
|
|
1082
|
+
|
|
1083
|
+
parsed_properties["id"] = parsed_properties["title"] = product_id
|
|
1084
|
+
|
|
1085
|
+
# update downloadLink and orderLink
|
|
1086
|
+
parsed_properties["_dc_qs"] = quote_plus(qs)
|
|
1087
|
+
if parsed_properties["downloadLink"] != "Not Available":
|
|
1088
|
+
parsed_properties["downloadLink"] += f"?{qs}"
|
|
1089
|
+
|
|
1090
|
+
# parse metadata needing downloadLink
|
|
1091
|
+
dl_path = Fields("downloadLink")
|
|
1092
|
+
dl_path_from_root = Child(Root(), dl_path)
|
|
1093
|
+
for param, mapping in self.config.metadata_mapping.items():
|
|
1094
|
+
if dl_path in mapping or dl_path_from_root in mapping:
|
|
1095
|
+
parsed_properties.update(
|
|
1096
|
+
properties_from_json(parsed_properties, {param: mapping})
|
|
1097
|
+
)
|
|
1098
|
+
|
|
1099
|
+
# use product_type_config as default properties
|
|
1100
|
+
parsed_properties = dict(
|
|
1101
|
+
getattr(self.config, "product_type_config", {}),
|
|
1102
|
+
**parsed_properties,
|
|
1103
|
+
)
|
|
1104
|
+
|
|
1105
|
+
product = EOProduct(
|
|
1106
|
+
provider=self.provider,
|
|
1107
|
+
productType=product_type,
|
|
1108
|
+
properties=parsed_properties,
|
|
1109
|
+
)
|
|
1110
|
+
|
|
1111
|
+
return [
|
|
1112
|
+
product,
|
|
1113
|
+
]
|
|
540
1114
|
|
|
541
|
-
|
|
542
|
-
|
|
1115
|
+
def count_hits(
|
|
1116
|
+
self, count_url: Optional[str] = None, result_type: Optional[str] = None
|
|
1117
|
+
) -> int:
|
|
1118
|
+
"""Count method that will always return 1.
|
|
1119
|
+
|
|
1120
|
+
:param count_url: not used, only here because this method overwrites count_hits from the parent class
|
|
1121
|
+
:param result_type: not used, only here because this method overwrites count_hits from the parent class
|
|
1122
|
+
:return: always 1
|
|
1123
|
+
"""
|
|
1124
|
+
return 1
|
|
1125
|
+
|
|
1126
|
+
|
|
1127
|
+
class MeteoblueSearch(ECMWFSearch):
|
|
1128
|
+
"""MeteoblueSearch search plugin.
|
|
1129
|
+
|
|
1130
|
+
This plugin, which inherits from :class:`~eodag.plugins.search.build_search_result.ECMWFSearch`,
|
|
1131
|
+
performs a POST request and uses its result to build a single :class:`~eodag.api.search_result.SearchResult`
|
|
1132
|
+
object.
|
|
1133
|
+
|
|
1134
|
+
The available configuration parameters are inherited from parent classes, with some a particularity
|
|
1135
|
+
for pagination for this plugin.
|
|
1136
|
+
|
|
1137
|
+
:param provider: An eodag providers configuration dictionary
|
|
1138
|
+
:param config: Search plugin configuration:
|
|
1139
|
+
|
|
1140
|
+
* :attr:`~eodag.config.PluginConfig.pagination` (:class:`~eodag.config.PluginConfig.Pagination`)
|
|
1141
|
+
(**mandatory**): The configuration of how the pagination is done on the provider. For
|
|
1142
|
+
this plugin it has the node:
|
|
1143
|
+
|
|
1144
|
+
* :attr:`~eodag.config.PluginConfig.Pagination.next_page_query_obj` (``str``): The
|
|
1145
|
+
additional parameters needed to perform search. These parameters won't be included in
|
|
1146
|
+
the result. This must be a json dict formatted like ``{{"foo":"bar"}}`` because it
|
|
1147
|
+
will be passed to a :meth:`str.format` method before being loaded as json.
|
|
1148
|
+
"""
|
|
1149
|
+
|
|
1150
|
+
def collect_search_urls(
|
|
1151
|
+
self,
|
|
1152
|
+
prep: PreparedSearch = PreparedSearch(),
|
|
1153
|
+
**kwargs: Any,
|
|
1154
|
+
) -> tuple[list[str], int]:
|
|
1155
|
+
"""Wraps PostJsonSearch.collect_search_urls to force product count to 1
|
|
1156
|
+
|
|
1157
|
+
:param prep: :class:`~eodag.plugins.search.PreparedSearch` object containing information for the search
|
|
1158
|
+
:param kwargs: keyword arguments used in the search
|
|
1159
|
+
:return: list of search url and number of results
|
|
1160
|
+
"""
|
|
1161
|
+
urls, _ = super().collect_search_urls(prep, **kwargs)
|
|
1162
|
+
return urls, 1
|
|
1163
|
+
|
|
1164
|
+
def do_search(
|
|
1165
|
+
self, prep: PreparedSearch = PreparedSearch(items_per_page=None), **kwargs: Any
|
|
1166
|
+
) -> list[dict[str, Any]]:
|
|
1167
|
+
"""Perform the actual search request, and return result in a single element.
|
|
1168
|
+
|
|
1169
|
+
:param prep: :class:`~eodag.plugins.search.PreparedSearch` object containing information for the search
|
|
1170
|
+
:param kwargs: keyword arguments to be used in the search
|
|
1171
|
+
:return: list containing the results from the provider in json format
|
|
1172
|
+
"""
|
|
1173
|
+
|
|
1174
|
+
prep.url = prep.search_urls[0]
|
|
1175
|
+
prep.info_message = f"Sending search request: {prep.url}"
|
|
1176
|
+
prep.exception_message = (
|
|
1177
|
+
f"Skipping error while searching for {self.provider}"
|
|
1178
|
+
f" {self.__class__.__name__} instance"
|
|
1179
|
+
)
|
|
1180
|
+
response = self._request(prep)
|
|
1181
|
+
|
|
1182
|
+
return [response.json()]
|
|
1183
|
+
|
|
1184
|
+
def build_query_string(
|
|
1185
|
+
self, product_type: str, **kwargs: Any
|
|
1186
|
+
) -> tuple[dict[str, Any], str]:
|
|
1187
|
+
"""Build The query string using the search parameters
|
|
1188
|
+
|
|
1189
|
+
:param product_type: product type id
|
|
1190
|
+
:param kwargs: keyword arguments to be used in the query string
|
|
1191
|
+
:return: formatted query params and encode query string
|
|
1192
|
+
"""
|
|
1193
|
+
return QueryStringSearch.build_query_string(
|
|
1194
|
+
self, product_type=product_type, **kwargs
|
|
1195
|
+
)
|
|
1196
|
+
|
|
1197
|
+
|
|
1198
|
+
class WekeoECMWFSearch(ECMWFSearch):
|
|
1199
|
+
"""
|
|
1200
|
+
WekeoECMWFSearch search plugin.
|
|
1201
|
+
|
|
1202
|
+
This plugin, which inherits from :class:`~eodag.plugins.search.build_search_result.ECMWFSearch`,
|
|
1203
|
+
performs a POST request and uses its result to build a single :class:`~eodag.api.search_result.SearchResult`
|
|
1204
|
+
object. In contrast to ECMWFSearch or MeteoblueSearch, the products are only build with information
|
|
1205
|
+
returned by the provider.
|
|
1206
|
+
|
|
1207
|
+
The available configuration parameters are inherited from parent classes, with some a particularity
|
|
1208
|
+
for pagination for this plugin.
|
|
1209
|
+
|
|
1210
|
+
:param provider: An eodag providers configuration dictionary
|
|
1211
|
+
:param config: Search plugin configuration:
|
|
1212
|
+
|
|
1213
|
+
* :attr:`~eodag.config.PluginConfig.pagination` (:class:`~eodag.config.PluginConfig.Pagination`)
|
|
1214
|
+
(**mandatory**): The configuration of how the pagination is done on the provider. For
|
|
1215
|
+
this plugin it has the node:
|
|
1216
|
+
|
|
1217
|
+
* :attr:`~eodag.config.PluginConfig.Pagination.next_page_query_obj` (``str``): The
|
|
1218
|
+
additional parameters needed to perform search. These parameters won't be included in
|
|
1219
|
+
the result. This must be a json dict formatted like ``{{"foo":"bar"}}`` because it
|
|
1220
|
+
will be passed to a :meth:`str.format` method before being loaded as json.
|
|
1221
|
+
"""
|
|
1222
|
+
|
|
1223
|
+
def normalize_results(
|
|
1224
|
+
self, results: RawSearchResult, **kwargs: Any
|
|
1225
|
+
) -> list[EOProduct]:
|
|
1226
|
+
"""Build :class:`~eodag.api.product._product.EOProduct` from provider result
|
|
1227
|
+
|
|
1228
|
+
:param results: Raw provider result as single dict in list
|
|
1229
|
+
:param kwargs: Search arguments
|
|
1230
|
+
:returns: list of single :class:`~eodag.api.product._product.EOProduct`
|
|
1231
|
+
"""
|
|
1232
|
+
|
|
1233
|
+
# formating of orderLink requires access to the productType value.
|
|
1234
|
+
results.data = [
|
|
1235
|
+
{**result, **results.product_type_def_params} for result in results
|
|
1236
|
+
]
|
|
1237
|
+
|
|
1238
|
+
normalized = QueryStringSearch.normalize_results(self, results, **kwargs)
|
|
1239
|
+
|
|
1240
|
+
if not normalized:
|
|
1241
|
+
return normalized
|
|
1242
|
+
|
|
1243
|
+
query_params_encoded = quote_plus(orjson.dumps(results.query_params))
|
|
1244
|
+
for product in normalized:
|
|
1245
|
+
properties = {**product.properties, **results.query_params}
|
|
1246
|
+
properties["_dc_qs"] = query_params_encoded
|
|
1247
|
+
product.properties = {ecmwf_format(k): v for k, v in properties.items()}
|
|
1248
|
+
|
|
1249
|
+
return normalized
|
|
1250
|
+
|
|
1251
|
+
def do_search(self, *args: Any, **kwargs: Any) -> list[dict[str, Any]]:
|
|
1252
|
+
"""Should perform the actual search request.
|
|
1253
|
+
|
|
1254
|
+
:param args: arguments to be used in the search
|
|
1255
|
+
:param kwargs: keyword arguments to be used in the search
|
|
1256
|
+
:return: list containing the results from the provider in json format
|
|
1257
|
+
"""
|
|
1258
|
+
return QueryStringSearch.do_search(self, *args, **kwargs)
|
|
1259
|
+
|
|
1260
|
+
def build_query_string(
|
|
1261
|
+
self, product_type: str, **kwargs: Any
|
|
1262
|
+
) -> tuple[dict[str, Any], str]:
|
|
1263
|
+
"""Build The query string using the search parameters
|
|
1264
|
+
|
|
1265
|
+
:param product_type: product type id
|
|
1266
|
+
:param kwargs: keyword arguments to be used in the query string
|
|
1267
|
+
:return: formatted query params and encode query string
|
|
1268
|
+
"""
|
|
1269
|
+
# Reorder kwargs to make sure year/month/day/time if set overwrite default datetime.
|
|
1270
|
+
# strip_quotes to remove duplicated quotes like "'1_1'" produced by convertors like to_geojson.
|
|
1271
|
+
priority_keys = [
|
|
1272
|
+
"startTimeFromAscendingNode",
|
|
1273
|
+
"completionTimeFromAscendingNode",
|
|
1274
|
+
]
|
|
1275
|
+
ordered_kwargs = {k: kwargs[k] for k in priority_keys if k in kwargs}
|
|
1276
|
+
ordered_kwargs.update({k: strip_quotes(v) for k, v in kwargs.items()})
|
|
1277
|
+
|
|
1278
|
+
return QueryStringSearch.build_query_string(
|
|
1279
|
+
self, product_type=product_type, **ordered_kwargs
|
|
1280
|
+
)
|