eodag 3.0.1__py3-none-any.whl → 3.1.0b1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- eodag/api/core.py +116 -86
- eodag/api/product/_assets.py +6 -6
- eodag/api/product/_product.py +18 -18
- eodag/api/product/metadata_mapping.py +39 -11
- eodag/cli.py +22 -1
- eodag/config.py +14 -14
- eodag/plugins/apis/ecmwf.py +37 -14
- eodag/plugins/apis/usgs.py +5 -5
- eodag/plugins/authentication/openid_connect.py +2 -2
- eodag/plugins/authentication/token.py +37 -6
- eodag/plugins/crunch/filter_property.py +2 -3
- eodag/plugins/download/aws.py +11 -12
- eodag/plugins/download/base.py +30 -39
- eodag/plugins/download/creodias_s3.py +29 -0
- eodag/plugins/download/http.py +144 -152
- eodag/plugins/download/s3rest.py +5 -7
- eodag/plugins/search/base.py +73 -25
- eodag/plugins/search/build_search_result.py +1047 -310
- eodag/plugins/search/creodias_s3.py +25 -19
- eodag/plugins/search/data_request_search.py +1 -1
- eodag/plugins/search/qssearch.py +51 -139
- eodag/resources/ext_product_types.json +1 -1
- eodag/resources/product_types.yml +391 -32
- eodag/resources/providers.yml +678 -1744
- eodag/rest/core.py +92 -62
- eodag/rest/server.py +31 -4
- eodag/rest/types/eodag_search.py +6 -0
- eodag/rest/types/queryables.py +5 -6
- eodag/rest/utils/__init__.py +3 -0
- eodag/types/__init__.py +56 -15
- eodag/types/download_args.py +2 -2
- eodag/types/queryables.py +180 -72
- eodag/types/whoosh.py +126 -0
- eodag/utils/__init__.py +71 -10
- eodag/utils/exceptions.py +27 -20
- eodag/utils/repr.py +65 -6
- eodag/utils/requests.py +11 -11
- {eodag-3.0.1.dist-info → eodag-3.1.0b1.dist-info}/METADATA +76 -76
- {eodag-3.0.1.dist-info → eodag-3.1.0b1.dist-info}/RECORD +43 -44
- {eodag-3.0.1.dist-info → eodag-3.1.0b1.dist-info}/WHEEL +1 -1
- {eodag-3.0.1.dist-info → eodag-3.1.0b1.dist-info}/entry_points.txt +3 -2
- eodag/utils/constraints.py +0 -244
- {eodag-3.0.1.dist-info → eodag-3.1.0b1.dist-info}/LICENSE +0 -0
- {eodag-3.0.1.dist-info → eodag-3.1.0b1.dist-info}/top_level.txt +0 -0
|
@@ -17,9 +17,12 @@
|
|
|
17
17
|
# limitations under the License.
|
|
18
18
|
from __future__ import annotations
|
|
19
19
|
|
|
20
|
+
import functools
|
|
20
21
|
import hashlib
|
|
21
22
|
import logging
|
|
22
|
-
|
|
23
|
+
import re
|
|
24
|
+
from collections import OrderedDict
|
|
25
|
+
from datetime import datetime, timedelta
|
|
23
26
|
from typing import (
|
|
24
27
|
TYPE_CHECKING,
|
|
25
28
|
Annotated,
|
|
@@ -29,8 +32,8 @@ from typing import (
|
|
|
29
32
|
Optional,
|
|
30
33
|
Set,
|
|
31
34
|
Tuple,
|
|
35
|
+
Union,
|
|
32
36
|
cast,
|
|
33
|
-
get_args,
|
|
34
37
|
)
|
|
35
38
|
from urllib.parse import quote_plus, unquote_plus
|
|
36
39
|
|
|
@@ -39,235 +42,330 @@ import orjson
|
|
|
39
42
|
from dateutil.parser import isoparse
|
|
40
43
|
from dateutil.tz import tzutc
|
|
41
44
|
from jsonpath_ng import Child, Fields, Root
|
|
42
|
-
from pydantic import
|
|
45
|
+
from pydantic import Field
|
|
43
46
|
from pydantic.fields import FieldInfo
|
|
47
|
+
from requests.auth import AuthBase
|
|
48
|
+
from shapely.geometry.base import BaseGeometry
|
|
49
|
+
from typing_extensions import get_args
|
|
44
50
|
|
|
45
51
|
from eodag.api.product import EOProduct
|
|
46
52
|
from eodag.api.product.metadata_mapping import (
|
|
47
53
|
NOT_AVAILABLE,
|
|
48
54
|
NOT_MAPPED,
|
|
49
|
-
|
|
55
|
+
format_metadata,
|
|
56
|
+
format_query_params,
|
|
50
57
|
mtd_cfg_as_conversion_and_querypath,
|
|
51
58
|
properties_from_json,
|
|
52
59
|
)
|
|
53
60
|
from eodag.api.search_result import RawSearchResult
|
|
54
61
|
from eodag.plugins.search import PreparedSearch
|
|
55
|
-
from eodag.plugins.search.
|
|
56
|
-
from eodag.
|
|
57
|
-
from eodag.types import
|
|
58
|
-
from eodag.types.queryables import CommonQueryables
|
|
62
|
+
from eodag.plugins.search.qssearch import PostJsonSearch, QueryStringSearch
|
|
63
|
+
from eodag.types import json_field_definition_to_python
|
|
64
|
+
from eodag.types.queryables import Queryables
|
|
59
65
|
from eodag.utils import (
|
|
60
|
-
|
|
66
|
+
HTTP_REQ_TIMEOUT,
|
|
61
67
|
deepcopy,
|
|
62
68
|
dict_items_recursive_sort,
|
|
63
69
|
get_geometry_from_various,
|
|
64
|
-
|
|
65
|
-
from eodag.utils.constraints import (
|
|
66
|
-
fetch_constraints,
|
|
67
|
-
get_constraint_queryables_with_additional_params,
|
|
70
|
+
is_range_in_range,
|
|
68
71
|
)
|
|
69
72
|
from eodag.utils.exceptions import ValidationError
|
|
73
|
+
from eodag.utils.requests import fetch_json
|
|
70
74
|
|
|
71
75
|
if TYPE_CHECKING:
|
|
72
76
|
from eodag.config import PluginConfig
|
|
73
77
|
|
|
74
78
|
logger = logging.getLogger("eodag.search.build_search_result")
|
|
75
79
|
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
|
|
80
|
-
|
|
81
|
-
|
|
82
|
-
|
|
83
|
-
|
|
84
|
-
|
|
85
|
-
|
|
86
|
-
|
|
87
|
-
|
|
88
|
-
|
|
89
|
-
|
|
90
|
-
|
|
91
|
-
|
|
92
|
-
|
|
93
|
-
|
|
80
|
+
# keywords from ECMWF keyword database + "dataset" (not part of database but exists)
|
|
81
|
+
# database: https://confluence.ecmwf.int/display/UDOC/Keywords+in+MARS+and+Dissemination+requests
|
|
82
|
+
ECMWF_KEYWORDS = [
|
|
83
|
+
"dataset",
|
|
84
|
+
"accuracy",
|
|
85
|
+
"activity",
|
|
86
|
+
"anoffset",
|
|
87
|
+
"bitmap",
|
|
88
|
+
"block",
|
|
89
|
+
"channel",
|
|
90
|
+
"class",
|
|
91
|
+
"database",
|
|
92
|
+
"date",
|
|
93
|
+
"diagnostic",
|
|
94
|
+
"direction",
|
|
95
|
+
"domain",
|
|
96
|
+
"duplicates",
|
|
97
|
+
"expect",
|
|
98
|
+
"expver",
|
|
99
|
+
"fcmonth",
|
|
100
|
+
"fcperiod",
|
|
101
|
+
"fieldset",
|
|
102
|
+
"filter",
|
|
103
|
+
"format",
|
|
104
|
+
"frame",
|
|
105
|
+
"frequency",
|
|
106
|
+
"generation",
|
|
107
|
+
"grid",
|
|
108
|
+
"hdate",
|
|
109
|
+
"ident",
|
|
110
|
+
"interpolation",
|
|
111
|
+
"intgrid",
|
|
112
|
+
"iteration",
|
|
113
|
+
"latitude",
|
|
114
|
+
"levelist",
|
|
115
|
+
"levtype",
|
|
116
|
+
"longitude",
|
|
117
|
+
"lsm",
|
|
118
|
+
"method",
|
|
119
|
+
"number",
|
|
120
|
+
"obsgroup",
|
|
121
|
+
"obstype",
|
|
122
|
+
"origin",
|
|
123
|
+
"packing",
|
|
124
|
+
"padding",
|
|
125
|
+
"param",
|
|
126
|
+
"priority",
|
|
127
|
+
"product",
|
|
128
|
+
"range",
|
|
129
|
+
"realization",
|
|
130
|
+
"refdate",
|
|
131
|
+
"reference",
|
|
132
|
+
"reportype",
|
|
133
|
+
"repres",
|
|
134
|
+
"resolution",
|
|
135
|
+
"rotation",
|
|
136
|
+
"section",
|
|
137
|
+
"source",
|
|
138
|
+
"step",
|
|
139
|
+
"stream",
|
|
140
|
+
"system",
|
|
141
|
+
"target",
|
|
142
|
+
"time",
|
|
143
|
+
"truncation",
|
|
144
|
+
"type",
|
|
145
|
+
"use",
|
|
146
|
+
]
|
|
147
|
+
|
|
148
|
+
# additional keywords from copernicus services
|
|
149
|
+
COP_DS_KEYWORDS = [
|
|
150
|
+
"aerosol_type",
|
|
151
|
+
"altitude",
|
|
152
|
+
"product_type",
|
|
153
|
+
"band",
|
|
154
|
+
"cdr_type",
|
|
155
|
+
"data_format",
|
|
156
|
+
"dataset_type",
|
|
157
|
+
"day",
|
|
158
|
+
"download_format",
|
|
159
|
+
"ensemble_member",
|
|
160
|
+
"experiment",
|
|
161
|
+
"forcing_type",
|
|
162
|
+
"gcm",
|
|
163
|
+
"hday",
|
|
164
|
+
"hmonth",
|
|
165
|
+
"horizontal_resolution",
|
|
166
|
+
"hydrological_model",
|
|
167
|
+
"hydrological_year",
|
|
168
|
+
"hyear",
|
|
169
|
+
"input_observations",
|
|
170
|
+
"leadtime_hour",
|
|
171
|
+
"leadtime_month",
|
|
172
|
+
"level",
|
|
173
|
+
"location",
|
|
174
|
+
"model",
|
|
175
|
+
"model_level",
|
|
176
|
+
"model_levels",
|
|
177
|
+
"month",
|
|
178
|
+
"nominal_day",
|
|
179
|
+
"originating_centre",
|
|
180
|
+
"period",
|
|
181
|
+
"pressure_level",
|
|
182
|
+
"processing_level",
|
|
183
|
+
"processing_type",
|
|
184
|
+
"product_version",
|
|
185
|
+
"quantity",
|
|
186
|
+
"rcm",
|
|
187
|
+
"region",
|
|
188
|
+
"release_version",
|
|
189
|
+
"satellite",
|
|
190
|
+
"sensor",
|
|
191
|
+
"sensor_and_algorithm",
|
|
192
|
+
"soil_level",
|
|
193
|
+
"sky_type",
|
|
194
|
+
"statistic",
|
|
195
|
+
"system_version",
|
|
196
|
+
"temporal_aggregation",
|
|
197
|
+
"time_aggregation",
|
|
198
|
+
"time_reference",
|
|
199
|
+
"time_step",
|
|
200
|
+
"variable",
|
|
201
|
+
"variable_type",
|
|
202
|
+
"version",
|
|
203
|
+
"year",
|
|
204
|
+
]
|
|
205
|
+
|
|
206
|
+
|
|
207
|
+
def keywords_to_mdt(
|
|
208
|
+
keywords: List[str], prefix: Optional[str] = None
|
|
209
|
+
) -> Dict[str, Any]:
|
|
94
210
|
"""
|
|
211
|
+
Make metadata mapping dict from a list of keywords
|
|
95
212
|
|
|
96
|
-
|
|
97
|
-
|
|
98
|
-
|
|
99
|
-
"""Count method that will always return 1."""
|
|
100
|
-
return 1
|
|
213
|
+
prefix:keyword:
|
|
214
|
+
- keyword
|
|
215
|
+
- $."prefix:keyword"
|
|
101
216
|
|
|
102
|
-
|
|
103
|
-
|
|
104
|
-
|
|
105
|
-
|
|
106
|
-
) -> Tuple[List[str], int]:
|
|
107
|
-
"""Wraps PostJsonSearch.collect_search_urls to force product count to 1"""
|
|
108
|
-
urls, _ = super(BuildPostSearchResult, self).collect_search_urls(prep, **kwargs)
|
|
109
|
-
return urls, 1
|
|
217
|
+
>>> keywords_to_mdt(["month", "year"])
|
|
218
|
+
{'month': ['month', '$."month"'], 'year': ['year', '$."year"']}
|
|
219
|
+
>>> keywords_to_mdt(["month", "year"], "ecmwf")
|
|
220
|
+
{'ecmwf:month': ['month', '$."ecmwf:month"'], 'ecmwf:year': ['year', '$."ecmwf:year"']}
|
|
110
221
|
|
|
111
|
-
|
|
112
|
-
|
|
113
|
-
|
|
114
|
-
|
|
115
|
-
|
|
116
|
-
|
|
117
|
-
|
|
118
|
-
|
|
119
|
-
|
|
120
|
-
)
|
|
121
|
-
response = self._request(prep)
|
|
122
|
-
|
|
123
|
-
return [response.json()]
|
|
222
|
+
:param keywords: List of keywords to be converted
|
|
223
|
+
:param prefix: prefix to be added to the parameter in the mapping
|
|
224
|
+
:return: metadata mapping dict
|
|
225
|
+
"""
|
|
226
|
+
mdt: Dict[str, Any] = {}
|
|
227
|
+
for keyword in keywords:
|
|
228
|
+
key = f"{prefix}:{keyword}" if prefix else keyword
|
|
229
|
+
mdt[key] = [keyword, f'$."{key}"']
|
|
230
|
+
return mdt
|
|
124
231
|
|
|
125
|
-
def normalize_results(
|
|
126
|
-
self, results: RawSearchResult, **kwargs: Any
|
|
127
|
-
) -> List[EOProduct]:
|
|
128
|
-
"""Build :class:`~eodag.api.product._product.EOProduct` from provider result
|
|
129
232
|
|
|
130
|
-
|
|
131
|
-
|
|
132
|
-
:returns: list of single :class:`~eodag.api.product._product.EOProduct`
|
|
133
|
-
"""
|
|
134
|
-
product_type = kwargs.get("productType")
|
|
233
|
+
def strip_quotes(value: Any) -> Any:
|
|
234
|
+
"""Strip superfluous quotes from elements (added by mapping converter to_geojson).
|
|
135
235
|
|
|
136
|
-
|
|
236
|
+
>>> strip_quotes("'abc'")
|
|
237
|
+
'abc'
|
|
238
|
+
>>> strip_quotes(["'abc'", '"def'])
|
|
239
|
+
['abc', 'def']
|
|
137
240
|
|
|
138
|
-
|
|
139
|
-
|
|
140
|
-
|
|
141
|
-
|
|
142
|
-
|
|
143
|
-
|
|
144
|
-
|
|
145
|
-
|
|
146
|
-
|
|
147
|
-
|
|
148
|
-
|
|
149
|
-
|
|
150
|
-
|
|
151
|
-
|
|
152
|
-
|
|
153
|
-
|
|
154
|
-
|
|
155
|
-
|
|
156
|
-
|
|
157
|
-
|
|
158
|
-
|
|
159
|
-
|
|
160
|
-
|
|
161
|
-
|
|
162
|
-
|
|
163
|
-
|
|
164
|
-
|
|
165
|
-
|
|
166
|
-
|
|
167
|
-
|
|
168
|
-
|
|
169
|
-
|
|
170
|
-
|
|
241
|
+
:param value: value from which quotes should be removed (should be either str or list)
|
|
242
|
+
:return: value without quotes
|
|
243
|
+
:raises: NotImplementedError
|
|
244
|
+
"""
|
|
245
|
+
if isinstance(value, (list, tuple)):
|
|
246
|
+
return [strip_quotes(v) for v in value]
|
|
247
|
+
elif isinstance(value, dict):
|
|
248
|
+
raise NotImplementedError("Dict value is not supported.")
|
|
249
|
+
else:
|
|
250
|
+
return str(value).strip("'\"")
|
|
251
|
+
|
|
252
|
+
|
|
253
|
+
def _update_properties_from_element(
|
|
254
|
+
prop: Dict[str, Any], element: Dict[str, Any], values: List[str]
|
|
255
|
+
) -> None:
|
|
256
|
+
"""updates a property dict with the given values based on the information from the element dict
|
|
257
|
+
e.g. the type is set based on the type of the element
|
|
258
|
+
"""
|
|
259
|
+
# multichoice elements are transformed into array
|
|
260
|
+
if element["type"] in ("StringListWidget", "StringListArrayWidget"):
|
|
261
|
+
prop["type"] = "array"
|
|
262
|
+
if values:
|
|
263
|
+
prop["items"] = {"type": "string", "enum": sorted(values)}
|
|
264
|
+
|
|
265
|
+
# single choice elements are transformed into string
|
|
266
|
+
elif element["type"] in (
|
|
267
|
+
"StringChoiceWidget",
|
|
268
|
+
"DateRangeWidget",
|
|
269
|
+
"FreeformInputWidget",
|
|
270
|
+
):
|
|
271
|
+
prop["type"] = "string"
|
|
272
|
+
if values:
|
|
273
|
+
prop["enum"] = sorted(values)
|
|
274
|
+
|
|
275
|
+
# a bbox element
|
|
276
|
+
elif element["type"] in ["GeographicExtentWidget", "GeographicExtentMapWidget"]:
|
|
277
|
+
prop.update(
|
|
278
|
+
{
|
|
279
|
+
"type": "array",
|
|
280
|
+
"minItems": 4,
|
|
281
|
+
"additionalItems": False,
|
|
282
|
+
"items": [
|
|
283
|
+
{
|
|
284
|
+
"type": "number",
|
|
285
|
+
"maximum": 180,
|
|
286
|
+
"minimum": -180,
|
|
287
|
+
"description": "West border of the bounding box",
|
|
288
|
+
},
|
|
289
|
+
{
|
|
290
|
+
"type": "number",
|
|
291
|
+
"maximum": 90,
|
|
292
|
+
"minimum": -90,
|
|
293
|
+
"description": "South border of the bounding box",
|
|
294
|
+
},
|
|
295
|
+
{
|
|
296
|
+
"type": "number",
|
|
297
|
+
"maximum": 180,
|
|
298
|
+
"minimum": -180,
|
|
299
|
+
"description": "East border of the bounding box",
|
|
300
|
+
},
|
|
301
|
+
{
|
|
302
|
+
"type": "number",
|
|
303
|
+
"maximum": 90,
|
|
304
|
+
"minimum": -90,
|
|
305
|
+
"description": "North border of the bounding box",
|
|
306
|
+
},
|
|
307
|
+
],
|
|
308
|
+
}
|
|
171
309
|
)
|
|
172
310
|
|
|
173
|
-
|
|
174
|
-
|
|
175
|
-
|
|
176
|
-
|
|
177
|
-
qs = geojson.dumps(sorted_unpaginated_query_params)
|
|
178
|
-
|
|
179
|
-
query_hash = hashlib.sha1(str(qs).encode("UTF-8")).hexdigest()
|
|
311
|
+
# DateRangeWidget is a calendar date picker
|
|
312
|
+
if element["type"] == "DateRangeWidget":
|
|
313
|
+
prop["description"] = "date formatted like yyyy-mm-dd/yyyy-mm-dd"
|
|
180
314
|
|
|
181
|
-
|
|
182
|
-
|
|
183
|
-
result.update(results.product_type_def_params)
|
|
184
|
-
result = dict(result, **{k: v for k, v in kwargs.items() if v is not None})
|
|
315
|
+
if description := element.get("help"):
|
|
316
|
+
prop["description"] = description
|
|
185
317
|
|
|
186
|
-
# parse properties
|
|
187
|
-
parsed_properties = properties_from_json(
|
|
188
|
-
result,
|
|
189
|
-
self.config.metadata_mapping,
|
|
190
|
-
discovery_config=getattr(self.config, "discover_metadata", {}),
|
|
191
|
-
)
|
|
192
318
|
|
|
193
|
-
|
|
194
|
-
|
|
319
|
+
def ecmwf_format(v: str) -> str:
|
|
320
|
+
"""Add ECMWF prefix to value v if v is a ECMWF keyword."""
|
|
321
|
+
return "ecmwf:" + v if v in ECMWF_KEYWORDS + COP_DS_KEYWORDS else v
|
|
195
322
|
|
|
196
|
-
# build product id
|
|
197
|
-
id_prefix = (product_type or self.provider).upper()
|
|
198
|
-
product_id = "%s_%s_%s_%s" % (
|
|
199
|
-
id_prefix,
|
|
200
|
-
parsed_properties["startTimeFromAscendingNode"]
|
|
201
|
-
.split("T")[0]
|
|
202
|
-
.replace("-", ""),
|
|
203
|
-
parsed_properties["completionTimeFromAscendingNode"]
|
|
204
|
-
.split("T")[0]
|
|
205
|
-
.replace("-", ""),
|
|
206
|
-
query_hash,
|
|
207
|
-
)
|
|
208
|
-
parsed_properties["id"] = parsed_properties["title"] = product_id
|
|
209
323
|
|
|
210
|
-
|
|
211
|
-
|
|
212
|
-
if parsed_properties["downloadLink"] != "Not Available":
|
|
213
|
-
parsed_properties["downloadLink"] += f"?{qs}"
|
|
324
|
+
class ECMWFSearch(PostJsonSearch):
|
|
325
|
+
"""ECMWF search plugin.
|
|
214
326
|
|
|
215
|
-
|
|
216
|
-
dl_path = Fields("downloadLink")
|
|
217
|
-
dl_path_from_root = Child(Root(), dl_path)
|
|
218
|
-
for param, mapping in self.config.metadata_mapping.items():
|
|
219
|
-
if dl_path in mapping or dl_path_from_root in mapping:
|
|
220
|
-
parsed_properties.update(
|
|
221
|
-
properties_from_json(parsed_properties, {param: mapping})
|
|
222
|
-
)
|
|
223
|
-
|
|
224
|
-
# use product_type_config as default properties
|
|
225
|
-
parsed_properties = dict(
|
|
226
|
-
getattr(self.config, "product_type_config", {}),
|
|
227
|
-
**parsed_properties,
|
|
228
|
-
)
|
|
229
|
-
|
|
230
|
-
product = EOProduct(
|
|
231
|
-
provider=self.provider,
|
|
232
|
-
productType=product_type,
|
|
233
|
-
properties=parsed_properties,
|
|
234
|
-
)
|
|
235
|
-
|
|
236
|
-
return [
|
|
237
|
-
product,
|
|
238
|
-
]
|
|
239
|
-
|
|
240
|
-
|
|
241
|
-
class BuildSearchResult(BuildPostSearchResult):
|
|
242
|
-
"""BuildSearchResult search plugin.
|
|
243
|
-
|
|
244
|
-
This plugin builds a single :class:`~eodag.api.search_result.SearchResult` object
|
|
327
|
+
This plugin builds a :class:`~eodag.api.search_result.SearchResult` containing a single product
|
|
245
328
|
using given query parameters as product properties.
|
|
246
329
|
|
|
247
|
-
The available configuration parameters inherits from parent classes
|
|
248
|
-
|
|
249
|
-
:class:`~eodag.plugins.search.qssearch.PostJsonSearch` and
|
|
250
|
-
:class:`~eodag.plugins.search.qssearch.QueryStringSearch`), with particularly for this plugin:
|
|
330
|
+
The available configuration parameters inherits from parent classes, with some particular parameters
|
|
331
|
+
for this plugin.
|
|
251
332
|
|
|
252
|
-
:param provider:
|
|
333
|
+
:param provider: An eodag providers configuration dictionary
|
|
253
334
|
:param config: Search plugin configuration:
|
|
254
335
|
|
|
255
|
-
* :attr:`~eodag.config.PluginConfig.
|
|
256
|
-
|
|
257
|
-
|
|
258
|
-
|
|
336
|
+
* :attr:`~eodag.config.PluginConfig.remove_from_query` (``List[str]``): List of parameters
|
|
337
|
+
used to parse metadata but that must not be included to the query
|
|
338
|
+
* :attr:`~eodag.config.PluginConfig.end_date_excluded` (``bool``): Set to `False` if
|
|
339
|
+
provider does not include end date to search
|
|
340
|
+
* :attr:`~eodag.config.PluginConfig.discover_queryables`
|
|
341
|
+
(:class:`~eodag.config.PluginConfig.DiscoverQueryables`): configuration to fetch the queryables from a
|
|
342
|
+
provider queryables endpoint; It has the following keys:
|
|
343
|
+
|
|
344
|
+
* :attr:`~eodag.config.PluginConfig.DiscoverQueryables.fetch_url` (``str``): url to fetch the queryables valid
|
|
345
|
+
for all product types
|
|
346
|
+
* :attr:`~eodag.config.PluginConfig.DiscoverQueryables.product_type_fetch_url` (``str``): url to fetch the
|
|
347
|
+
queryables for a specific product type
|
|
348
|
+
* :attr:`~eodag.config.PluginConfig.DiscoverQueryables.constraints_url` (``str``): url of the constraint file
|
|
349
|
+
used to build queryables
|
|
259
350
|
"""
|
|
260
351
|
|
|
261
352
|
def __init__(self, provider: str, config: PluginConfig) -> None:
|
|
262
|
-
#
|
|
263
|
-
|
|
353
|
+
# cache fetching method
|
|
354
|
+
self.fetch_data = functools.lru_cache()(self._fetch_data)
|
|
355
|
+
|
|
356
|
+
config.metadata_mapping = {
|
|
357
|
+
**keywords_to_mdt(ECMWF_KEYWORDS + COP_DS_KEYWORDS, "ecmwf"),
|
|
358
|
+
**config.metadata_mapping,
|
|
359
|
+
}
|
|
360
|
+
|
|
361
|
+
super().__init__(provider, config)
|
|
264
362
|
|
|
265
363
|
self.config.__dict__.setdefault("api_endpoint", "")
|
|
266
364
|
|
|
267
365
|
# needed by QueryStringSearch.build_query_string / format_free_text_search
|
|
268
366
|
self.config.__dict__.setdefault("free_text_search_operations", {})
|
|
269
367
|
# needed for compatibility
|
|
270
|
-
self.config.
|
|
368
|
+
self.config.pagination.setdefault("next_page_query_obj", "{{}}")
|
|
271
369
|
|
|
272
370
|
# parse jsonpath on init: product type specific metadata-mapping
|
|
273
371
|
for product_type in self.config.products.keys():
|
|
@@ -305,7 +403,13 @@ class BuildSearchResult(BuildPostSearchResult):
|
|
|
305
403
|
] = product_type_metadata_mapping
|
|
306
404
|
|
|
307
405
|
def do_search(self, *args: Any, **kwargs: Any) -> List[Dict[str, Any]]:
|
|
308
|
-
"""Should perform the actual search request.
|
|
406
|
+
"""Should perform the actual search request.
|
|
407
|
+
|
|
408
|
+
:param args: arguments to be used in the search
|
|
409
|
+
:param kwargs: keyword arguments to be used in the search
|
|
410
|
+
:return: list containing the results from the provider in json format
|
|
411
|
+
"""
|
|
412
|
+
# no real search. We fake it all
|
|
309
413
|
return [{}]
|
|
310
414
|
|
|
311
415
|
def query(
|
|
@@ -313,37 +417,55 @@ class BuildSearchResult(BuildPostSearchResult):
|
|
|
313
417
|
prep: PreparedSearch = PreparedSearch(),
|
|
314
418
|
**kwargs: Any,
|
|
315
419
|
) -> Tuple[List[EOProduct], Optional[int]]:
|
|
316
|
-
"""Build ready-to-download SearchResult
|
|
420
|
+
"""Build ready-to-download SearchResult
|
|
317
421
|
|
|
318
|
-
|
|
422
|
+
:param prep: :class:`~eodag.plugins.search.PreparedSearch` object containing information needed for the search
|
|
423
|
+
:param kwargs: keyword arguments to be used in the search
|
|
424
|
+
:returns: list of products and number of products (optional)
|
|
425
|
+
"""
|
|
426
|
+
product_type = prep.product_type
|
|
427
|
+
if not product_type:
|
|
428
|
+
product_type = kwargs.get("productType", None)
|
|
429
|
+
self._preprocess_search_params(kwargs, product_type)
|
|
430
|
+
result, num_items = super().query(prep, **kwargs)
|
|
431
|
+
if prep.count and not num_items:
|
|
432
|
+
num_items = 1
|
|
319
433
|
|
|
320
|
-
return
|
|
434
|
+
return result, num_items
|
|
321
435
|
|
|
322
436
|
def clear(self) -> None:
|
|
323
437
|
"""Clear search context"""
|
|
324
|
-
|
|
438
|
+
super().clear()
|
|
325
439
|
|
|
326
440
|
def build_query_string(
|
|
327
441
|
self, product_type: str, **kwargs: Any
|
|
328
442
|
) -> Tuple[Dict[str, Any], str]:
|
|
329
|
-
"""Build The query string using the search parameters
|
|
443
|
+
"""Build The query string using the search parameters
|
|
444
|
+
|
|
445
|
+
:param product_type: product type id
|
|
446
|
+
:param kwargs: keyword arguments to be used in the query string
|
|
447
|
+
:return: formatted query params and encode query string
|
|
448
|
+
"""
|
|
330
449
|
# parse kwargs as properties as they might be needed to build the query
|
|
331
450
|
parsed_properties = properties_from_json(
|
|
332
451
|
kwargs,
|
|
333
452
|
self.config.metadata_mapping,
|
|
334
453
|
)
|
|
335
454
|
available_properties = {
|
|
336
|
-
|
|
455
|
+
# We strip values of superfluous quotes (added by mapping converter to_geojson).
|
|
456
|
+
k: strip_quotes(v)
|
|
337
457
|
for k, v in parsed_properties.items()
|
|
338
458
|
if v not in [NOT_AVAILABLE, NOT_MAPPED]
|
|
339
459
|
}
|
|
340
460
|
|
|
341
461
|
# build and return the query
|
|
342
|
-
return
|
|
343
|
-
|
|
462
|
+
return super().build_query_string(
|
|
463
|
+
product_type=product_type, **available_properties
|
|
344
464
|
)
|
|
345
465
|
|
|
346
|
-
def _preprocess_search_params(
|
|
466
|
+
def _preprocess_search_params(
|
|
467
|
+
self, params: Dict[str, Any], product_type: Optional[str]
|
|
468
|
+
) -> None:
|
|
347
469
|
"""Preprocess search parameters before making a request to the CDS API.
|
|
348
470
|
|
|
349
471
|
This method is responsible for checking and updating the provided search parameters
|
|
@@ -352,6 +474,7 @@ class BuildSearchResult(BuildPostSearchResult):
|
|
|
352
474
|
in the input parameters, default values or values from the configuration are used.
|
|
353
475
|
|
|
354
476
|
:param params: Search parameters to be preprocessed.
|
|
477
|
+
:param product_type: (optional) product type id
|
|
355
478
|
"""
|
|
356
479
|
_dc_qs = params.get("_dc_qs", None)
|
|
357
480
|
if _dc_qs is not None:
|
|
@@ -378,63 +501,43 @@ class BuildSearchResult(BuildPostSearchResult):
|
|
|
378
501
|
non_none_params = {k: v for k, v in params.items() if v}
|
|
379
502
|
|
|
380
503
|
# productType
|
|
381
|
-
dataset = params.get("dataset", None)
|
|
504
|
+
dataset = params.get("ecmwf:dataset", None)
|
|
382
505
|
params["productType"] = non_none_params.get("productType", dataset)
|
|
383
506
|
|
|
384
507
|
# dates
|
|
385
|
-
|
|
386
|
-
|
|
387
|
-
|
|
388
|
-
).replace(
|
|
389
|
-
"Z", "+00:00"
|
|
390
|
-
) # before 3.11
|
|
391
|
-
)
|
|
392
|
-
|
|
393
|
-
default_end_from_cfg = self.config.products.get(params["productType"], {}).get(
|
|
394
|
-
"_default_end_date", None
|
|
395
|
-
)
|
|
396
|
-
default_end_str = (
|
|
397
|
-
default_end_from_cfg
|
|
398
|
-
or (
|
|
399
|
-
datetime.now(timezone.utc)
|
|
400
|
-
if params.get("startTimeFromAscendingNode")
|
|
401
|
-
else mission_start_dt + timedelta(days=1)
|
|
402
|
-
).isoformat()
|
|
403
|
-
)
|
|
404
|
-
|
|
405
|
-
params["startTimeFromAscendingNode"] = non_none_params.get(
|
|
406
|
-
"startTimeFromAscendingNode", mission_start_dt.isoformat()
|
|
407
|
-
)
|
|
408
|
-
params["completionTimeFromAscendingNode"] = non_none_params.get(
|
|
409
|
-
"completionTimeFromAscendingNode", default_end_str
|
|
410
|
-
)
|
|
508
|
+
# check if default dates have to be added
|
|
509
|
+
if getattr(self.config, "dates_required", False):
|
|
510
|
+
self._check_date_params(params, product_type)
|
|
411
511
|
|
|
412
512
|
# adapt end date if it is midnight
|
|
413
|
-
|
|
414
|
-
|
|
415
|
-
|
|
416
|
-
end_date = datetime.strptime(
|
|
417
|
-
params["completionTimeFromAscendingNode"], "%Y-%m-%dT%H:%M:%SZ"
|
|
418
|
-
)
|
|
419
|
-
end_date = end_date.replace(tzinfo=tzutc())
|
|
420
|
-
except ValueError:
|
|
513
|
+
if "completionTimeFromAscendingNode" in params:
|
|
514
|
+
end_date_excluded = getattr(self.config, "end_date_excluded", True)
|
|
515
|
+
is_datetime = True
|
|
421
516
|
try:
|
|
422
517
|
end_date = datetime.strptime(
|
|
423
|
-
params["completionTimeFromAscendingNode"], "%Y-%m-%dT%H:%M:%
|
|
518
|
+
params["completionTimeFromAscendingNode"], "%Y-%m-%dT%H:%M:%SZ"
|
|
424
519
|
)
|
|
425
520
|
end_date = end_date.replace(tzinfo=tzutc())
|
|
426
521
|
except ValueError:
|
|
427
|
-
|
|
428
|
-
|
|
429
|
-
|
|
430
|
-
|
|
431
|
-
|
|
432
|
-
|
|
433
|
-
|
|
434
|
-
|
|
435
|
-
|
|
436
|
-
|
|
437
|
-
|
|
522
|
+
try:
|
|
523
|
+
end_date = datetime.strptime(
|
|
524
|
+
params["completionTimeFromAscendingNode"],
|
|
525
|
+
"%Y-%m-%dT%H:%M:%S.%fZ",
|
|
526
|
+
)
|
|
527
|
+
end_date = end_date.replace(tzinfo=tzutc())
|
|
528
|
+
except ValueError:
|
|
529
|
+
end_date = isoparse(params["completionTimeFromAscendingNode"])
|
|
530
|
+
is_datetime = False
|
|
531
|
+
start_date = isoparse(params["startTimeFromAscendingNode"])
|
|
532
|
+
if (
|
|
533
|
+
not end_date_excluded
|
|
534
|
+
and is_datetime
|
|
535
|
+
and end_date > start_date
|
|
536
|
+
and end_date
|
|
537
|
+
== end_date.replace(hour=0, minute=0, second=0, microsecond=0)
|
|
538
|
+
):
|
|
539
|
+
end_date += timedelta(days=-1)
|
|
540
|
+
params["completionTimeFromAscendingNode"] = end_date.isoformat()
|
|
438
541
|
|
|
439
542
|
# geometry
|
|
440
543
|
if "geometry" in params:
|
|
@@ -449,94 +552,728 @@ class BuildSearchResult(BuildPostSearchResult):
|
|
|
449
552
|
arguments)
|
|
450
553
|
:returns: fetched queryable parameters dict
|
|
451
554
|
"""
|
|
452
|
-
|
|
453
|
-
|
|
454
|
-
|
|
455
|
-
|
|
456
|
-
|
|
457
|
-
|
|
555
|
+
product_type = kwargs.pop("productType")
|
|
556
|
+
product_type_config = self.config.products.get(product_type, {})
|
|
557
|
+
provider_product_type = (
|
|
558
|
+
product_type_config.get("ecmwf:dataset", None)
|
|
559
|
+
or product_type_config["productType"]
|
|
560
|
+
)
|
|
561
|
+
if "start" in kwargs:
|
|
562
|
+
kwargs["startTimeFromAscendingNode"] = kwargs.pop("start")
|
|
563
|
+
if "end" in kwargs:
|
|
564
|
+
kwargs["completionTimeFromAscendingNode"] = kwargs.pop("end")
|
|
565
|
+
|
|
566
|
+
# extract default datetime
|
|
567
|
+
processed_kwargs = deepcopy(kwargs)
|
|
568
|
+
self._preprocess_search_params(processed_kwargs, product_type)
|
|
569
|
+
|
|
570
|
+
constraints_url = format_metadata(
|
|
571
|
+
getattr(self.config, "discover_queryables", {}).get("constraints_url", ""),
|
|
572
|
+
**kwargs,
|
|
573
|
+
)
|
|
574
|
+
constraints: List[Dict[str, Any]] = self.fetch_data(constraints_url)
|
|
575
|
+
|
|
576
|
+
form_url = format_metadata(
|
|
577
|
+
getattr(self.config, "discover_queryables", {}).get("form_url", ""),
|
|
578
|
+
**kwargs,
|
|
579
|
+
)
|
|
580
|
+
form = self.fetch_data(form_url)
|
|
458
581
|
|
|
459
|
-
|
|
460
|
-
|
|
582
|
+
formated_kwargs = self.format_as_provider_keyword(
|
|
583
|
+
product_type, processed_kwargs
|
|
461
584
|
)
|
|
462
|
-
|
|
585
|
+
# we re-apply kwargs input to consider override of year, month, day and time.
|
|
586
|
+
for key in kwargs:
|
|
587
|
+
if key.startswith("ecmwf:"):
|
|
588
|
+
formated_kwargs[key.replace("ecmwf:", "")] = kwargs[key]
|
|
589
|
+
elif key in (
|
|
590
|
+
"startTimeFromAscendingNode",
|
|
591
|
+
"completionTimeFromAscendingNode",
|
|
592
|
+
"geom",
|
|
593
|
+
):
|
|
594
|
+
formated_kwargs[key] = kwargs[key]
|
|
595
|
+
else:
|
|
596
|
+
raise ValidationError(
|
|
597
|
+
f"{key} is not a queryable parameter for {self.provider}"
|
|
598
|
+
)
|
|
599
|
+
|
|
600
|
+
# we use non empty kwargs as default to integrate user inputs
|
|
601
|
+
# it is needed because pydantic json schema does not represent "value"
|
|
602
|
+
# but only "default"
|
|
603
|
+
non_empty_formated: Dict[str, Any] = {
|
|
604
|
+
k: v
|
|
605
|
+
for k, v in formated_kwargs.items()
|
|
606
|
+
if v and (not isinstance(v, list) or all(v))
|
|
607
|
+
}
|
|
608
|
+
non_empty_kwargs: Dict[str, Any] = {
|
|
609
|
+
k: v
|
|
610
|
+
for k, v in processed_kwargs.items()
|
|
611
|
+
if v and (not isinstance(v, list) or all(v))
|
|
612
|
+
}
|
|
613
|
+
|
|
614
|
+
required_keywords: Set[str] = set()
|
|
615
|
+
|
|
616
|
+
# calculate available values
|
|
617
|
+
if constraints:
|
|
618
|
+
# Apply constraints filtering
|
|
619
|
+
available_values = self.available_values_from_constraints(
|
|
620
|
+
constraints,
|
|
621
|
+
non_empty_formated,
|
|
622
|
+
form_keywords=[f["name"] for f in form],
|
|
623
|
+
)
|
|
624
|
+
|
|
625
|
+
# Pre-compute the required keywords (present in all constraint dicts)
|
|
626
|
+
# when form, required keywords are extracted directly from form
|
|
627
|
+
if not form:
|
|
628
|
+
required_keywords = set(constraints[0].keys())
|
|
629
|
+
for constraint in constraints[1:]:
|
|
630
|
+
required_keywords.intersection_update(constraint.keys())
|
|
631
|
+
else:
|
|
632
|
+
values_url = getattr(self.config, "available_values_url", "")
|
|
633
|
+
if not values_url:
|
|
634
|
+
return self.queryables_from_metadata_mapping(product_type)
|
|
635
|
+
if "{" in values_url:
|
|
636
|
+
values_url = values_url.format(productType=provider_product_type)
|
|
637
|
+
data = self.fetch_data(values_url)
|
|
638
|
+
available_values = data["constraints"]
|
|
639
|
+
required_keywords = data.get("required", [])
|
|
640
|
+
|
|
641
|
+
# To check if all keywords are queryable parameters, we check if they are in the
|
|
642
|
+
# available values or the product type config (available values calculated from the
|
|
643
|
+
# constraints might not include all queryables)
|
|
644
|
+
for keyword in kwargs:
|
|
645
|
+
if (
|
|
646
|
+
keyword
|
|
647
|
+
not in available_values.keys()
|
|
648
|
+
| product_type_config.keys()
|
|
649
|
+
| {
|
|
650
|
+
"startTimeFromAscendingNode",
|
|
651
|
+
"completionTimeFromAscendingNode",
|
|
652
|
+
"geom",
|
|
653
|
+
}
|
|
654
|
+
and keyword.replace("ecmwf:", "") not in available_values
|
|
655
|
+
):
|
|
656
|
+
raise ValidationError(f"{keyword} is not a queryable parameter")
|
|
657
|
+
|
|
658
|
+
# generate queryables
|
|
659
|
+
if form:
|
|
660
|
+
queryables = self.queryables_by_form(
|
|
661
|
+
form,
|
|
662
|
+
available_values,
|
|
663
|
+
non_empty_formated,
|
|
664
|
+
)
|
|
665
|
+
else:
|
|
666
|
+
queryables = self.queryables_by_values(
|
|
667
|
+
available_values, list(required_keywords), non_empty_kwargs
|
|
668
|
+
)
|
|
669
|
+
|
|
670
|
+
# ecmwf:date is replaced by start and end.
|
|
671
|
+
# start and end filters are supported whenever combinations of "year", "month", "day" filters exist
|
|
463
672
|
if (
|
|
464
|
-
|
|
465
|
-
|
|
673
|
+
queryables.pop("ecmwf:date", None)
|
|
674
|
+
or "ecmwf:year" in queryables
|
|
675
|
+
or "ecmwf:hyear" in queryables
|
|
466
676
|
):
|
|
467
|
-
|
|
468
|
-
|
|
677
|
+
queryables.update(
|
|
678
|
+
{
|
|
679
|
+
"start": Queryables.get_with_default(
|
|
680
|
+
"start", non_empty_kwargs.get("startTimeFromAscendingNode")
|
|
681
|
+
),
|
|
682
|
+
"end": Queryables.get_with_default(
|
|
683
|
+
"end",
|
|
684
|
+
non_empty_kwargs.get("completionTimeFromAscendingNode"),
|
|
685
|
+
),
|
|
686
|
+
}
|
|
469
687
|
)
|
|
470
688
|
|
|
471
|
-
#
|
|
472
|
-
|
|
473
|
-
|
|
474
|
-
|
|
689
|
+
# area is geom in EODAG.
|
|
690
|
+
if queryables.pop("area", None):
|
|
691
|
+
queryables["geom"] = Annotated[
|
|
692
|
+
Union[str, Dict[str, float], BaseGeometry],
|
|
693
|
+
Field(
|
|
694
|
+
None,
|
|
695
|
+
description="Read EODAG documentation for all supported geometry format.",
|
|
696
|
+
),
|
|
697
|
+
]
|
|
475
698
|
|
|
476
|
-
|
|
699
|
+
return queryables
|
|
477
700
|
|
|
478
|
-
|
|
479
|
-
|
|
480
|
-
|
|
481
|
-
|
|
482
|
-
|
|
483
|
-
|
|
484
|
-
|
|
485
|
-
|
|
486
|
-
|
|
487
|
-
|
|
488
|
-
|
|
489
|
-
|
|
490
|
-
|
|
491
|
-
|
|
492
|
-
|
|
493
|
-
|
|
494
|
-
|
|
495
|
-
|
|
496
|
-
|
|
497
|
-
|
|
498
|
-
|
|
499
|
-
|
|
500
|
-
|
|
701
|
+
def available_values_from_constraints(
|
|
702
|
+
self,
|
|
703
|
+
constraints: list[Dict[str, Any]],
|
|
704
|
+
input_keywords: Dict[str, Any],
|
|
705
|
+
form_keywords: List[str],
|
|
706
|
+
) -> Dict[str, List[str]]:
|
|
707
|
+
"""
|
|
708
|
+
Filter constraints using input_keywords. Return list of available queryables.
|
|
709
|
+
All constraint entries must have the same parameters.
|
|
710
|
+
|
|
711
|
+
:param constraints: list of constraints received from the provider
|
|
712
|
+
:param input_keywords: dict of input parameters given by the user
|
|
713
|
+
:param form_keywords: list of keyword names from the provider form endpoint
|
|
714
|
+
:return: dict with available values for each parameter
|
|
715
|
+
"""
|
|
716
|
+
# get ordered constraint keywords
|
|
717
|
+
constraints_keywords = list(
|
|
718
|
+
OrderedDict.fromkeys(k for c in constraints for k in c.keys())
|
|
719
|
+
)
|
|
720
|
+
|
|
721
|
+
# prepare ordered input keywords formatted as provider's keywords
|
|
722
|
+
# required to filter with constraints
|
|
723
|
+
ordered_keywords = (
|
|
724
|
+
[kw for kw in form_keywords if kw in constraints_keywords]
|
|
725
|
+
if form_keywords
|
|
726
|
+
else constraints_keywords
|
|
727
|
+
)
|
|
728
|
+
|
|
729
|
+
# filter constraint entries matching input keyword values
|
|
730
|
+
filtered_constraints: List[Dict[str, Any]]
|
|
731
|
+
|
|
732
|
+
parsed_keywords: List[str] = []
|
|
733
|
+
for keyword in ordered_keywords:
|
|
734
|
+
values = input_keywords.get(keyword)
|
|
735
|
+
|
|
736
|
+
if values is None:
|
|
737
|
+
parsed_keywords.append(keyword)
|
|
738
|
+
continue
|
|
739
|
+
|
|
740
|
+
# we only compare list of strings.
|
|
741
|
+
if isinstance(values, dict):
|
|
742
|
+
raise ValidationError(
|
|
743
|
+
f"Parameter value as object is not supported: {keyword}={values}"
|
|
744
|
+
)
|
|
745
|
+
filter_v = values if isinstance(values, (list, tuple)) else [values]
|
|
746
|
+
|
|
747
|
+
# We convert every single value to a list of string
|
|
748
|
+
# We strip values of superfluous quotes (added by mapping converter to_geojson).
|
|
749
|
+
# ECMWF accept values with /to/. We need to split it to an array
|
|
750
|
+
# ECMWF accept values in format val1/val2. We need to split it to an array
|
|
751
|
+
sep = re.compile(r"/to/|/")
|
|
752
|
+
filter_v = [i for v in filter_v for i in sep.split(strip_quotes(v))]
|
|
753
|
+
|
|
754
|
+
# special handling for time 0000 converted to 0 by pre-formating with metadata_mapping
|
|
755
|
+
if keyword.split(":")[-1] == "time":
|
|
756
|
+
filter_v = ["0000" if str(v) == "0" else v for v in filter_v]
|
|
757
|
+
|
|
758
|
+
# Collect missing values to report errors
|
|
759
|
+
missing_values = set(filter_v)
|
|
760
|
+
|
|
761
|
+
# Filter constraints and check for missing values
|
|
762
|
+
filtered_constraints = []
|
|
763
|
+
for entry in constraints:
|
|
764
|
+
# Filter based on the presence of any value in filter_v
|
|
765
|
+
entry_values = entry.get(keyword, [])
|
|
766
|
+
|
|
767
|
+
# date constraint may be intervals. We identify intervals with a "/" in the value
|
|
768
|
+
# we assume that if the first value is an interval, all values are intervals
|
|
769
|
+
present_values = []
|
|
770
|
+
if keyword == "date" and "/" in entry[keyword][0]:
|
|
771
|
+
if any(is_range_in_range(x, values[0]) for x in entry[keyword]):
|
|
772
|
+
present_values = filter_v
|
|
773
|
+
else:
|
|
774
|
+
present_values = [
|
|
775
|
+
value for value in filter_v if value in entry_values
|
|
776
|
+
]
|
|
777
|
+
|
|
778
|
+
# Remove present values from the missing_values set
|
|
779
|
+
missing_values -= set(present_values)
|
|
780
|
+
|
|
781
|
+
if present_values:
|
|
782
|
+
filtered_constraints.append(entry)
|
|
783
|
+
|
|
784
|
+
# raise an error as no constraint entry matched the input keywords
|
|
785
|
+
# raise an error if one value from input is not allowed
|
|
786
|
+
if not filtered_constraints or missing_values:
|
|
787
|
+
allowed_values = list(
|
|
788
|
+
{value for c in constraints for value in c.get(keyword, [])}
|
|
789
|
+
)
|
|
790
|
+
# restore ecmwf: prefix before raising error
|
|
791
|
+
keyword = f"ecmwf:{keyword}"
|
|
792
|
+
|
|
793
|
+
all_keywords_str = ""
|
|
794
|
+
if len(parsed_keywords) > 1:
|
|
795
|
+
keywords = [
|
|
796
|
+
f"ecmwf:{k}={pk}"
|
|
797
|
+
for k in parsed_keywords
|
|
798
|
+
if (pk := input_keywords.get(k))
|
|
799
|
+
]
|
|
800
|
+
all_keywords_str = f" with {', '.join(keywords)}"
|
|
801
|
+
|
|
802
|
+
raise ValidationError(
|
|
803
|
+
f"{keyword}={values} is not available"
|
|
804
|
+
f"{all_keywords_str}."
|
|
805
|
+
f" Allowed values are {', '.join(allowed_values)}."
|
|
806
|
+
)
|
|
807
|
+
|
|
808
|
+
parsed_keywords.append(keyword)
|
|
809
|
+
constraints = filtered_constraints
|
|
810
|
+
|
|
811
|
+
available_values: Dict[str, Any] = {k: set() for k in ordered_keywords}
|
|
812
|
+
|
|
813
|
+
# we aggregate the constraint entries left
|
|
814
|
+
for entry in constraints:
|
|
815
|
+
for key, value in entry.items():
|
|
816
|
+
available_values[key].update(value)
|
|
817
|
+
|
|
818
|
+
return {k: list(v) for k, v in available_values.items()}
|
|
819
|
+
|
|
820
|
+
def queryables_by_form(
|
|
821
|
+
self,
|
|
822
|
+
form: List[Dict[str, Any]],
|
|
823
|
+
available_values: Dict[str, List[str]],
|
|
824
|
+
defaults: Dict[str, Any],
|
|
825
|
+
) -> Dict[str, Annotated[Any, FieldInfo]]:
|
|
826
|
+
"""
|
|
827
|
+
Generate Annotated field definitions from form entries and available values
|
|
828
|
+
Used by Copernicus services like cop_cds, cop_ads, cop_ewds.
|
|
829
|
+
|
|
830
|
+
:param form: data fetched from the form endpoint of the provider
|
|
831
|
+
:param available_values: available values for each parameter
|
|
832
|
+
:param defaults: default values for the parameters
|
|
833
|
+
:return: dict of annotated queryables
|
|
834
|
+
"""
|
|
835
|
+
queryables: Dict[str, Annotated[Any, FieldInfo]] = {}
|
|
836
|
+
|
|
837
|
+
required_list: List[str] = []
|
|
838
|
+
for element in form:
|
|
839
|
+
name: str = element["name"]
|
|
840
|
+
|
|
841
|
+
# those are not parameter elements.
|
|
842
|
+
if name in ("area_group", "global", "warning", "licences"):
|
|
843
|
+
continue
|
|
844
|
+
if "type" not in element or element["type"] == "FreeEditionWidget":
|
|
845
|
+
continue
|
|
846
|
+
|
|
847
|
+
# ordering done by id -> set id to high value if not present -> element will be last
|
|
848
|
+
if "id" not in element:
|
|
849
|
+
element["id"] = 100
|
|
850
|
+
|
|
851
|
+
prop = {"title": element.get("label", name)}
|
|
852
|
+
|
|
853
|
+
details = element.get("details", {})
|
|
854
|
+
|
|
855
|
+
# add values from form if keyword was not in constraints
|
|
856
|
+
values = (
|
|
857
|
+
available_values[name]
|
|
858
|
+
if name in available_values
|
|
859
|
+
else details.get("values")
|
|
501
860
|
)
|
|
502
|
-
|
|
503
|
-
|
|
504
|
-
|
|
505
|
-
|
|
506
|
-
|
|
507
|
-
|
|
508
|
-
|
|
509
|
-
|
|
510
|
-
|
|
511
|
-
|
|
512
|
-
|
|
513
|
-
if
|
|
514
|
-
|
|
515
|
-
|
|
861
|
+
|
|
862
|
+
# updates the properties with the values given based on the information from the element
|
|
863
|
+
_update_properties_from_element(prop, element, values)
|
|
864
|
+
|
|
865
|
+
default = defaults.get(name)
|
|
866
|
+
|
|
867
|
+
if details:
|
|
868
|
+
fields = details.get("fields")
|
|
869
|
+
if fields and (comment := fields[0].get("comment")):
|
|
870
|
+
prop["description"] = comment
|
|
871
|
+
|
|
872
|
+
if d := details.get("default"):
|
|
873
|
+
default = default or (d[0] if fields else d)
|
|
874
|
+
|
|
875
|
+
if name == "area" and isinstance(default, dict):
|
|
876
|
+
default = list(default.values())
|
|
877
|
+
|
|
878
|
+
if default:
|
|
879
|
+
# We strip values of superfluous quotes (addded by mapping converter to_geojson).
|
|
880
|
+
default = strip_quotes(default)
|
|
881
|
+
|
|
882
|
+
# sometimes form returns default as array instead of string
|
|
883
|
+
if default and prop["type"] == "string" and isinstance(default, list):
|
|
884
|
+
default = ",".join(default)
|
|
885
|
+
|
|
886
|
+
is_required = bool(element.get("required"))
|
|
887
|
+
if is_required:
|
|
888
|
+
required_list.append(name)
|
|
889
|
+
|
|
890
|
+
queryables[ecmwf_format(name)] = Annotated[
|
|
891
|
+
get_args(
|
|
892
|
+
json_field_definition_to_python(
|
|
893
|
+
prop,
|
|
894
|
+
default_value=default,
|
|
895
|
+
required=is_required,
|
|
516
896
|
)
|
|
517
|
-
|
|
518
|
-
|
|
519
|
-
|
|
520
|
-
|
|
521
|
-
|
|
522
|
-
|
|
897
|
+
)
|
|
898
|
+
]
|
|
899
|
+
|
|
900
|
+
return queryables
|
|
901
|
+
|
|
902
|
+
def queryables_by_values(
|
|
903
|
+
self,
|
|
904
|
+
available_values: Dict[str, List[str]],
|
|
905
|
+
required_keywords: List[str],
|
|
906
|
+
defaults: Dict[str, Any],
|
|
907
|
+
) -> Dict[str, Annotated[Any, FieldInfo]]:
|
|
908
|
+
"""
|
|
909
|
+
Generate Annotated field definitions from available values.
|
|
910
|
+
Used by ECMWF data providers like dedt_lumi.
|
|
911
|
+
|
|
912
|
+
:param available_values: available values for each parameter
|
|
913
|
+
:param required_keywords: list of required parameters
|
|
914
|
+
:param defaults: default values for the parameters
|
|
915
|
+
:return: dict of annotated queryables
|
|
916
|
+
"""
|
|
917
|
+
# Rename keywords from form with metadata mapping.
|
|
918
|
+
# Needed to map constraints like "xxxx" to eodag parameter "ecmwf:xxxx"
|
|
919
|
+
required = [ecmwf_format(k) for k in required_keywords]
|
|
920
|
+
|
|
921
|
+
queryables: Dict[str, Annotated[Any, FieldInfo]] = {}
|
|
922
|
+
for name, values in available_values.items():
|
|
923
|
+
# Rename keywords from form with metadata mapping.
|
|
924
|
+
# Needed to map constraints like "xxxx" to eodag parameter "ecmwf:xxxx"
|
|
925
|
+
key = ecmwf_format(name)
|
|
926
|
+
|
|
927
|
+
default = defaults.get(key)
|
|
928
|
+
|
|
929
|
+
queryables[key] = Annotated[
|
|
930
|
+
get_args(
|
|
931
|
+
json_field_definition_to_python(
|
|
932
|
+
{"type": "string", "title": name, "enum": values},
|
|
933
|
+
default_value=strip_quotes(default) if default else None,
|
|
934
|
+
required=bool(key in required),
|
|
523
935
|
)
|
|
936
|
+
)
|
|
937
|
+
]
|
|
938
|
+
|
|
939
|
+
return queryables
|
|
940
|
+
|
|
941
|
+
def format_as_provider_keyword(
|
|
942
|
+
self, product_type: str, properties: Dict[str, Any]
|
|
943
|
+
) -> Dict[str, Any]:
|
|
944
|
+
"""Return provider equivalent keyword names from EODAG keywords.
|
|
945
|
+
|
|
946
|
+
:param product_type: product type id
|
|
947
|
+
:param properties: dict of properties to be formatted
|
|
948
|
+
:return: dict of formatted properties
|
|
949
|
+
"""
|
|
950
|
+
parsed_properties = properties_from_json(
|
|
951
|
+
properties,
|
|
952
|
+
self.config.metadata_mapping,
|
|
953
|
+
)
|
|
954
|
+
available_properties = {
|
|
955
|
+
k: v
|
|
956
|
+
for k, v in parsed_properties.items()
|
|
957
|
+
if v not in [NOT_AVAILABLE, NOT_MAPPED]
|
|
958
|
+
}
|
|
959
|
+
return format_query_params(product_type, self.config, available_properties)
|
|
960
|
+
|
|
961
|
+
def _fetch_data(self, url: str) -> Any:
|
|
962
|
+
"""
|
|
963
|
+
fetches from a provider elements like constraints or forms.
|
|
964
|
+
|
|
965
|
+
:param url: url from which the constraints can be fetched
|
|
966
|
+
:returns: json file content fetched from the provider
|
|
967
|
+
"""
|
|
968
|
+
if not url:
|
|
969
|
+
return []
|
|
970
|
+
|
|
971
|
+
auth = (
|
|
972
|
+
self.auth
|
|
973
|
+
if hasattr(self, "auth") and isinstance(self.auth, AuthBase)
|
|
974
|
+
else None
|
|
975
|
+
)
|
|
976
|
+
timeout = getattr(self.config, "timeout", HTTP_REQ_TIMEOUT)
|
|
977
|
+
return fetch_json(url, auth=auth, timeout=timeout)
|
|
524
978
|
|
|
525
|
-
|
|
526
|
-
|
|
527
|
-
|
|
528
|
-
|
|
529
|
-
|
|
979
|
+
def normalize_results(
|
|
980
|
+
self, results: RawSearchResult, **kwargs: Any
|
|
981
|
+
) -> List[EOProduct]:
|
|
982
|
+
"""Build :class:`~eodag.api.product._product.EOProduct` from provider result
|
|
983
|
+
|
|
984
|
+
:param results: Raw provider result as single dict in list
|
|
985
|
+
:param kwargs: Search arguments
|
|
986
|
+
:returns: list of single :class:`~eodag.api.product._product.EOProduct`
|
|
987
|
+
"""
|
|
988
|
+
|
|
989
|
+
product_type = kwargs.get("productType")
|
|
990
|
+
|
|
991
|
+
result = results[0]
|
|
992
|
+
|
|
993
|
+
# datacube query string got from previous search
|
|
994
|
+
_dc_qs = kwargs.pop("_dc_qs", None)
|
|
995
|
+
if _dc_qs is not None:
|
|
996
|
+
qs = unquote_plus(unquote_plus(_dc_qs))
|
|
997
|
+
sorted_unpaginated_query_params = geojson.loads(qs)
|
|
998
|
+
else:
|
|
999
|
+
# update result with query parameters without pagination (or search-only params)
|
|
1000
|
+
if isinstance(
|
|
1001
|
+
self.config.pagination["next_page_query_obj"], str
|
|
1002
|
+
) and hasattr(results, "query_params_unpaginated"):
|
|
1003
|
+
unpaginated_query_params = results.query_params_unpaginated
|
|
1004
|
+
elif isinstance(self.config.pagination["next_page_query_obj"], str):
|
|
1005
|
+
next_page_query_obj = orjson.loads(
|
|
1006
|
+
self.config.pagination["next_page_query_obj"].format()
|
|
530
1007
|
)
|
|
531
|
-
|
|
1008
|
+
unpaginated_query_params = {
|
|
1009
|
+
k: v
|
|
1010
|
+
for k, v in results.query_params.items()
|
|
1011
|
+
if (k, v) not in next_page_query_obj.items()
|
|
1012
|
+
}
|
|
1013
|
+
else:
|
|
1014
|
+
unpaginated_query_params = self.query_params
|
|
1015
|
+
# query hash, will be used to build a product id
|
|
1016
|
+
sorted_unpaginated_query_params = dict_items_recursive_sort(
|
|
1017
|
+
unpaginated_query_params
|
|
1018
|
+
)
|
|
1019
|
+
|
|
1020
|
+
# use all available query_params to parse properties
|
|
1021
|
+
result = dict(
|
|
1022
|
+
result,
|
|
1023
|
+
**sorted_unpaginated_query_params,
|
|
1024
|
+
qs=sorted_unpaginated_query_params,
|
|
1025
|
+
)
|
|
1026
|
+
|
|
1027
|
+
# remove unwanted query params
|
|
1028
|
+
for param in getattr(self.config, "remove_from_query", []):
|
|
1029
|
+
sorted_unpaginated_query_params.pop(param, None)
|
|
1030
|
+
|
|
1031
|
+
qs = geojson.dumps(sorted_unpaginated_query_params)
|
|
1032
|
+
|
|
1033
|
+
query_hash = hashlib.sha1(str(qs).encode("UTF-8")).hexdigest()
|
|
1034
|
+
|
|
1035
|
+
# update result with product_type_def_params and search args if not None (and not auth)
|
|
1036
|
+
kwargs.pop("auth", None)
|
|
1037
|
+
result.update(results.product_type_def_params)
|
|
1038
|
+
result = dict(result, **{k: v for k, v in kwargs.items() if v is not None})
|
|
1039
|
+
|
|
1040
|
+
# parse properties
|
|
1041
|
+
parsed_properties = properties_from_json(
|
|
1042
|
+
result,
|
|
1043
|
+
self.config.metadata_mapping,
|
|
1044
|
+
discovery_config=getattr(self.config, "discover_metadata", {}),
|
|
1045
|
+
)
|
|
1046
|
+
|
|
1047
|
+
if not product_type:
|
|
1048
|
+
product_type = parsed_properties.get("productType", None)
|
|
1049
|
+
|
|
1050
|
+
# build product id
|
|
1051
|
+
id_prefix = (product_type or self.provider).upper()
|
|
1052
|
+
if (
|
|
1053
|
+
"startTimeFromAscendingNode" in parsed_properties
|
|
1054
|
+
and parsed_properties["startTimeFromAscendingNode"] != "Not Available"
|
|
1055
|
+
and "completionTimeFromAscendingNode" in parsed_properties
|
|
1056
|
+
and parsed_properties["completionTimeFromAscendingNode"] != "Not Available"
|
|
1057
|
+
):
|
|
1058
|
+
product_id = "%s_%s_%s_%s" % (
|
|
1059
|
+
id_prefix,
|
|
1060
|
+
parsed_properties["startTimeFromAscendingNode"]
|
|
1061
|
+
.split("T")[0]
|
|
1062
|
+
.replace("-", ""),
|
|
1063
|
+
parsed_properties["completionTimeFromAscendingNode"]
|
|
1064
|
+
.split("T")[0]
|
|
1065
|
+
.replace("-", ""),
|
|
1066
|
+
query_hash,
|
|
532
1067
|
)
|
|
533
|
-
|
|
534
|
-
|
|
535
|
-
|
|
536
|
-
|
|
537
|
-
|
|
1068
|
+
elif (
|
|
1069
|
+
"startTimeFromAscendingNode" in parsed_properties
|
|
1070
|
+
and parsed_properties["startTimeFromAscendingNode"] != "Not Available"
|
|
1071
|
+
):
|
|
1072
|
+
product_id = "%s_%s_%s" % (
|
|
1073
|
+
id_prefix,
|
|
1074
|
+
parsed_properties["startTimeFromAscendingNode"]
|
|
1075
|
+
.split("T")[0]
|
|
1076
|
+
.replace("-", ""),
|
|
1077
|
+
query_hash,
|
|
538
1078
|
)
|
|
539
|
-
|
|
1079
|
+
else:
|
|
1080
|
+
product_id = f"{id_prefix}_{query_hash}"
|
|
1081
|
+
|
|
1082
|
+
parsed_properties["id"] = parsed_properties["title"] = product_id
|
|
1083
|
+
|
|
1084
|
+
# update downloadLink and orderLink
|
|
1085
|
+
parsed_properties["_dc_qs"] = quote_plus(qs)
|
|
1086
|
+
if parsed_properties["downloadLink"] != "Not Available":
|
|
1087
|
+
parsed_properties["downloadLink"] += f"?{qs}"
|
|
1088
|
+
|
|
1089
|
+
# parse metadata needing downloadLink
|
|
1090
|
+
dl_path = Fields("downloadLink")
|
|
1091
|
+
dl_path_from_root = Child(Root(), dl_path)
|
|
1092
|
+
for param, mapping in self.config.metadata_mapping.items():
|
|
1093
|
+
if dl_path in mapping or dl_path_from_root in mapping:
|
|
1094
|
+
parsed_properties.update(
|
|
1095
|
+
properties_from_json(parsed_properties, {param: mapping})
|
|
1096
|
+
)
|
|
1097
|
+
|
|
1098
|
+
# use product_type_config as default properties
|
|
1099
|
+
parsed_properties = dict(
|
|
1100
|
+
getattr(self.config, "product_type_config", {}),
|
|
1101
|
+
**parsed_properties,
|
|
1102
|
+
)
|
|
1103
|
+
|
|
1104
|
+
product = EOProduct(
|
|
1105
|
+
provider=self.provider,
|
|
1106
|
+
productType=product_type,
|
|
1107
|
+
properties=parsed_properties,
|
|
1108
|
+
)
|
|
1109
|
+
|
|
1110
|
+
return [
|
|
1111
|
+
product,
|
|
1112
|
+
]
|
|
1113
|
+
|
|
1114
|
+
def count_hits(
|
|
1115
|
+
self, count_url: Optional[str] = None, result_type: Optional[str] = None
|
|
1116
|
+
) -> int:
|
|
1117
|
+
"""Count method that will always return 1.
|
|
1118
|
+
|
|
1119
|
+
:param count_url: not used, only here because this method overwrites count_hits from the parent class
|
|
1120
|
+
:param result_type: not used, only here because this method overwrites count_hits from the parent class
|
|
1121
|
+
:return: always 1
|
|
1122
|
+
"""
|
|
1123
|
+
return 1
|
|
1124
|
+
|
|
1125
|
+
|
|
1126
|
+
class MeteoblueSearch(ECMWFSearch):
|
|
1127
|
+
"""MeteoblueSearch search plugin.
|
|
1128
|
+
|
|
1129
|
+
This plugin, which inherits from :class:`~eodag.plugins.search.build_search_result.ECMWFSearch`,
|
|
1130
|
+
performs a POST request and uses its result to build a single :class:`~eodag.api.search_result.SearchResult`
|
|
1131
|
+
object.
|
|
1132
|
+
|
|
1133
|
+
The available configuration parameters are inherited from parent classes, with some a particularity
|
|
1134
|
+
for pagination for this plugin.
|
|
1135
|
+
|
|
1136
|
+
:param provider: An eodag providers configuration dictionary
|
|
1137
|
+
:param config: Search plugin configuration:
|
|
1138
|
+
|
|
1139
|
+
* :attr:`~eodag.config.PluginConfig.pagination` (:class:`~eodag.config.PluginConfig.Pagination`)
|
|
1140
|
+
(**mandatory**): The configuration of how the pagination is done on the provider. For
|
|
1141
|
+
this plugin it has the node:
|
|
1142
|
+
|
|
1143
|
+
* :attr:`~eodag.config.PluginConfig.Pagination.next_page_query_obj` (``str``): The
|
|
1144
|
+
additional parameters needed to perform search. These parameters won't be included in
|
|
1145
|
+
the result. This must be a json dict formatted like ``{{"foo":"bar"}}`` because it
|
|
1146
|
+
will be passed to a :meth:`str.format` method before being loaded as json.
|
|
1147
|
+
"""
|
|
1148
|
+
|
|
1149
|
+
def collect_search_urls(
|
|
1150
|
+
self,
|
|
1151
|
+
prep: PreparedSearch = PreparedSearch(),
|
|
1152
|
+
**kwargs: Any,
|
|
1153
|
+
) -> Tuple[List[str], int]:
|
|
1154
|
+
"""Wraps PostJsonSearch.collect_search_urls to force product count to 1
|
|
1155
|
+
|
|
1156
|
+
:param prep: :class:`~eodag.plugins.search.PreparedSearch` object containing information for the search
|
|
1157
|
+
:param kwargs: keyword arguments used in the search
|
|
1158
|
+
:return: list of search url and number of results
|
|
1159
|
+
"""
|
|
1160
|
+
urls, _ = super().collect_search_urls(prep, **kwargs)
|
|
1161
|
+
return urls, 1
|
|
1162
|
+
|
|
1163
|
+
def do_search(
|
|
1164
|
+
self, prep: PreparedSearch = PreparedSearch(items_per_page=None), **kwargs: Any
|
|
1165
|
+
) -> List[Dict[str, Any]]:
|
|
1166
|
+
"""Perform the actual search request, and return result in a single element.
|
|
1167
|
+
|
|
1168
|
+
:param prep: :class:`~eodag.plugins.search.PreparedSearch` object containing information for the search
|
|
1169
|
+
:param kwargs: keyword arguments to be used in the search
|
|
1170
|
+
:return: list containing the results from the provider in json format
|
|
1171
|
+
"""
|
|
540
1172
|
|
|
541
|
-
|
|
542
|
-
|
|
1173
|
+
prep.url = prep.search_urls[0]
|
|
1174
|
+
prep.info_message = f"Sending search request: {prep.url}"
|
|
1175
|
+
prep.exception_message = (
|
|
1176
|
+
f"Skipping error while searching for {self.provider}"
|
|
1177
|
+
f" {self.__class__.__name__} instance"
|
|
1178
|
+
)
|
|
1179
|
+
response = self._request(prep)
|
|
1180
|
+
|
|
1181
|
+
return [response.json()]
|
|
1182
|
+
|
|
1183
|
+
def build_query_string(
|
|
1184
|
+
self, product_type: str, **kwargs: Any
|
|
1185
|
+
) -> Tuple[Dict[str, Any], str]:
|
|
1186
|
+
"""Build The query string using the search parameters
|
|
1187
|
+
|
|
1188
|
+
:param product_type: product type id
|
|
1189
|
+
:param kwargs: keyword arguments to be used in the query string
|
|
1190
|
+
:return: formatted query params and encode query string
|
|
1191
|
+
"""
|
|
1192
|
+
return QueryStringSearch.build_query_string(
|
|
1193
|
+
self, product_type=product_type, **kwargs
|
|
1194
|
+
)
|
|
1195
|
+
|
|
1196
|
+
|
|
1197
|
+
class WekeoECMWFSearch(ECMWFSearch):
|
|
1198
|
+
"""
|
|
1199
|
+
WekeoECMWFSearch search plugin.
|
|
1200
|
+
|
|
1201
|
+
This plugin, which inherits from :class:`~eodag.plugins.search.build_search_result.ECMWFSearch`,
|
|
1202
|
+
performs a POST request and uses its result to build a single :class:`~eodag.api.search_result.SearchResult`
|
|
1203
|
+
object. In contrast to ECMWFSearch or MeteoblueSearch, the products are only build with information
|
|
1204
|
+
returned by the provider.
|
|
1205
|
+
|
|
1206
|
+
The available configuration parameters are inherited from parent classes, with some a particularity
|
|
1207
|
+
for pagination for this plugin.
|
|
1208
|
+
|
|
1209
|
+
:param provider: An eodag providers configuration dictionary
|
|
1210
|
+
:param config: Search plugin configuration:
|
|
1211
|
+
|
|
1212
|
+
* :attr:`~eodag.config.PluginConfig.pagination` (:class:`~eodag.config.PluginConfig.Pagination`)
|
|
1213
|
+
(**mandatory**): The configuration of how the pagination is done on the provider. For
|
|
1214
|
+
this plugin it has the node:
|
|
1215
|
+
|
|
1216
|
+
* :attr:`~eodag.config.PluginConfig.Pagination.next_page_query_obj` (``str``): The
|
|
1217
|
+
additional parameters needed to perform search. These parameters won't be included in
|
|
1218
|
+
the result. This must be a json dict formatted like ``{{"foo":"bar"}}`` because it
|
|
1219
|
+
will be passed to a :meth:`str.format` method before being loaded as json.
|
|
1220
|
+
"""
|
|
1221
|
+
|
|
1222
|
+
def normalize_results(
|
|
1223
|
+
self, results: RawSearchResult, **kwargs: Any
|
|
1224
|
+
) -> List[EOProduct]:
|
|
1225
|
+
"""Build :class:`~eodag.api.product._product.EOProduct` from provider result
|
|
1226
|
+
|
|
1227
|
+
:param results: Raw provider result as single dict in list
|
|
1228
|
+
:param kwargs: Search arguments
|
|
1229
|
+
:returns: list of single :class:`~eodag.api.product._product.EOProduct`
|
|
1230
|
+
"""
|
|
1231
|
+
|
|
1232
|
+
# formating of orderLink requires access to the productType value.
|
|
1233
|
+
results.data = [
|
|
1234
|
+
{**result, **results.product_type_def_params} for result in results
|
|
1235
|
+
]
|
|
1236
|
+
|
|
1237
|
+
normalized = QueryStringSearch.normalize_results(self, results, **kwargs)
|
|
1238
|
+
|
|
1239
|
+
if not normalized:
|
|
1240
|
+
return normalized
|
|
1241
|
+
|
|
1242
|
+
query_params_encoded = quote_plus(orjson.dumps(results.query_params))
|
|
1243
|
+
for product in normalized:
|
|
1244
|
+
properties = {**product.properties, **results.query_params}
|
|
1245
|
+
properties["_dc_qs"] = query_params_encoded
|
|
1246
|
+
product.properties = {ecmwf_format(k): v for k, v in properties.items()}
|
|
1247
|
+
|
|
1248
|
+
return normalized
|
|
1249
|
+
|
|
1250
|
+
def do_search(self, *args: Any, **kwargs: Any) -> List[Dict[str, Any]]:
|
|
1251
|
+
"""Should perform the actual search request.
|
|
1252
|
+
|
|
1253
|
+
:param args: arguments to be used in the search
|
|
1254
|
+
:param kwargs: keyword arguments to be used in the search
|
|
1255
|
+
:return: list containing the results from the provider in json format
|
|
1256
|
+
"""
|
|
1257
|
+
return QueryStringSearch.do_search(self, *args, **kwargs)
|
|
1258
|
+
|
|
1259
|
+
def build_query_string(
|
|
1260
|
+
self, product_type: str, **kwargs: Any
|
|
1261
|
+
) -> Tuple[Dict[str, Any], str]:
|
|
1262
|
+
"""Build The query string using the search parameters
|
|
1263
|
+
|
|
1264
|
+
:param product_type: product type id
|
|
1265
|
+
:param kwargs: keyword arguments to be used in the query string
|
|
1266
|
+
:return: formatted query params and encode query string
|
|
1267
|
+
"""
|
|
1268
|
+
# Reorder kwargs to make sure year/month/day/time if set overwrite default datetime.
|
|
1269
|
+
# strip_quotes to remove duplicated quotes like "'1_1'" produced by convertors like to_geojson.
|
|
1270
|
+
priority_keys = [
|
|
1271
|
+
"startTimeFromAscendingNode",
|
|
1272
|
+
"completionTimeFromAscendingNode",
|
|
1273
|
+
]
|
|
1274
|
+
ordered_kwargs = {k: kwargs[k] for k in priority_keys if k in kwargs}
|
|
1275
|
+
ordered_kwargs.update({k: strip_quotes(v) for k, v in kwargs.items()})
|
|
1276
|
+
|
|
1277
|
+
return QueryStringSearch.build_query_string(
|
|
1278
|
+
self, product_type=product_type, **ordered_kwargs
|
|
1279
|
+
)
|