eodag 3.0.1__py3-none-any.whl → 3.1.0b1__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (44) hide show
  1. eodag/api/core.py +116 -86
  2. eodag/api/product/_assets.py +6 -6
  3. eodag/api/product/_product.py +18 -18
  4. eodag/api/product/metadata_mapping.py +39 -11
  5. eodag/cli.py +22 -1
  6. eodag/config.py +14 -14
  7. eodag/plugins/apis/ecmwf.py +37 -14
  8. eodag/plugins/apis/usgs.py +5 -5
  9. eodag/plugins/authentication/openid_connect.py +2 -2
  10. eodag/plugins/authentication/token.py +37 -6
  11. eodag/plugins/crunch/filter_property.py +2 -3
  12. eodag/plugins/download/aws.py +11 -12
  13. eodag/plugins/download/base.py +30 -39
  14. eodag/plugins/download/creodias_s3.py +29 -0
  15. eodag/plugins/download/http.py +144 -152
  16. eodag/plugins/download/s3rest.py +5 -7
  17. eodag/plugins/search/base.py +73 -25
  18. eodag/plugins/search/build_search_result.py +1047 -310
  19. eodag/plugins/search/creodias_s3.py +25 -19
  20. eodag/plugins/search/data_request_search.py +1 -1
  21. eodag/plugins/search/qssearch.py +51 -139
  22. eodag/resources/ext_product_types.json +1 -1
  23. eodag/resources/product_types.yml +391 -32
  24. eodag/resources/providers.yml +678 -1744
  25. eodag/rest/core.py +92 -62
  26. eodag/rest/server.py +31 -4
  27. eodag/rest/types/eodag_search.py +6 -0
  28. eodag/rest/types/queryables.py +5 -6
  29. eodag/rest/utils/__init__.py +3 -0
  30. eodag/types/__init__.py +56 -15
  31. eodag/types/download_args.py +2 -2
  32. eodag/types/queryables.py +180 -72
  33. eodag/types/whoosh.py +126 -0
  34. eodag/utils/__init__.py +71 -10
  35. eodag/utils/exceptions.py +27 -20
  36. eodag/utils/repr.py +65 -6
  37. eodag/utils/requests.py +11 -11
  38. {eodag-3.0.1.dist-info → eodag-3.1.0b1.dist-info}/METADATA +76 -76
  39. {eodag-3.0.1.dist-info → eodag-3.1.0b1.dist-info}/RECORD +43 -44
  40. {eodag-3.0.1.dist-info → eodag-3.1.0b1.dist-info}/WHEEL +1 -1
  41. {eodag-3.0.1.dist-info → eodag-3.1.0b1.dist-info}/entry_points.txt +3 -2
  42. eodag/utils/constraints.py +0 -244
  43. {eodag-3.0.1.dist-info → eodag-3.1.0b1.dist-info}/LICENSE +0 -0
  44. {eodag-3.0.1.dist-info → eodag-3.1.0b1.dist-info}/top_level.txt +0 -0
@@ -17,9 +17,12 @@
17
17
  # limitations under the License.
18
18
  from __future__ import annotations
19
19
 
20
+ import functools
20
21
  import hashlib
21
22
  import logging
22
- from datetime import datetime, timedelta, timezone
23
+ import re
24
+ from collections import OrderedDict
25
+ from datetime import datetime, timedelta
23
26
  from typing import (
24
27
  TYPE_CHECKING,
25
28
  Annotated,
@@ -29,8 +32,8 @@ from typing import (
29
32
  Optional,
30
33
  Set,
31
34
  Tuple,
35
+ Union,
32
36
  cast,
33
- get_args,
34
37
  )
35
38
  from urllib.parse import quote_plus, unquote_plus
36
39
 
@@ -39,235 +42,330 @@ import orjson
39
42
  from dateutil.parser import isoparse
40
43
  from dateutil.tz import tzutc
41
44
  from jsonpath_ng import Child, Fields, Root
42
- from pydantic import create_model
45
+ from pydantic import Field
43
46
  from pydantic.fields import FieldInfo
47
+ from requests.auth import AuthBase
48
+ from shapely.geometry.base import BaseGeometry
49
+ from typing_extensions import get_args
44
50
 
45
51
  from eodag.api.product import EOProduct
46
52
  from eodag.api.product.metadata_mapping import (
47
53
  NOT_AVAILABLE,
48
54
  NOT_MAPPED,
49
- get_queryable_from_provider,
55
+ format_metadata,
56
+ format_query_params,
50
57
  mtd_cfg_as_conversion_and_querypath,
51
58
  properties_from_json,
52
59
  )
53
60
  from eodag.api.search_result import RawSearchResult
54
61
  from eodag.plugins.search import PreparedSearch
55
- from eodag.plugins.search.base import Search
56
- from eodag.plugins.search.qssearch import PostJsonSearch
57
- from eodag.types import json_field_definition_to_python, model_fields_to_annotated
58
- from eodag.types.queryables import CommonQueryables
62
+ from eodag.plugins.search.qssearch import PostJsonSearch, QueryStringSearch
63
+ from eodag.types import json_field_definition_to_python
64
+ from eodag.types.queryables import Queryables
59
65
  from eodag.utils import (
60
- DEFAULT_MISSION_START_DATE,
66
+ HTTP_REQ_TIMEOUT,
61
67
  deepcopy,
62
68
  dict_items_recursive_sort,
63
69
  get_geometry_from_various,
64
- )
65
- from eodag.utils.constraints import (
66
- fetch_constraints,
67
- get_constraint_queryables_with_additional_params,
70
+ is_range_in_range,
68
71
  )
69
72
  from eodag.utils.exceptions import ValidationError
73
+ from eodag.utils.requests import fetch_json
70
74
 
71
75
  if TYPE_CHECKING:
72
76
  from eodag.config import PluginConfig
73
77
 
74
78
  logger = logging.getLogger("eodag.search.build_search_result")
75
79
 
76
-
77
- class BuildPostSearchResult(PostJsonSearch):
78
- """BuildPostSearchResult search plugin.
79
-
80
- This plugin, which inherits from :class:`~eodag.plugins.search.qssearch.PostJsonSearch`,
81
- performs a POST request and uses its result to build a single :class:`~eodag.api.search_result.SearchResult`
82
- object.
83
-
84
- The available configuration parameters are inherited from parent classes
85
- (:class:`~eodag.plugins.search.qssearch.PostJsonSearch` and
86
- :class:`~eodag.plugins.search.qssearch.QueryStringSearch`), with particularly for this plugin:
87
-
88
- :param provider: provider name
89
- :param config: Search plugin configuration:
90
-
91
- * :attr:`~eodag.config.PluginConfig.remove_from_query` (``List[str]``): List of parameters
92
- used to parse metadata but that must not be included to the query
93
-
80
+ # keywords from ECMWF keyword database + "dataset" (not part of database but exists)
81
+ # database: https://confluence.ecmwf.int/display/UDOC/Keywords+in+MARS+and+Dissemination+requests
82
+ ECMWF_KEYWORDS = [
83
+ "dataset",
84
+ "accuracy",
85
+ "activity",
86
+ "anoffset",
87
+ "bitmap",
88
+ "block",
89
+ "channel",
90
+ "class",
91
+ "database",
92
+ "date",
93
+ "diagnostic",
94
+ "direction",
95
+ "domain",
96
+ "duplicates",
97
+ "expect",
98
+ "expver",
99
+ "fcmonth",
100
+ "fcperiod",
101
+ "fieldset",
102
+ "filter",
103
+ "format",
104
+ "frame",
105
+ "frequency",
106
+ "generation",
107
+ "grid",
108
+ "hdate",
109
+ "ident",
110
+ "interpolation",
111
+ "intgrid",
112
+ "iteration",
113
+ "latitude",
114
+ "levelist",
115
+ "levtype",
116
+ "longitude",
117
+ "lsm",
118
+ "method",
119
+ "number",
120
+ "obsgroup",
121
+ "obstype",
122
+ "origin",
123
+ "packing",
124
+ "padding",
125
+ "param",
126
+ "priority",
127
+ "product",
128
+ "range",
129
+ "realization",
130
+ "refdate",
131
+ "reference",
132
+ "reportype",
133
+ "repres",
134
+ "resolution",
135
+ "rotation",
136
+ "section",
137
+ "source",
138
+ "step",
139
+ "stream",
140
+ "system",
141
+ "target",
142
+ "time",
143
+ "truncation",
144
+ "type",
145
+ "use",
146
+ ]
147
+
148
+ # additional keywords from copernicus services
149
+ COP_DS_KEYWORDS = [
150
+ "aerosol_type",
151
+ "altitude",
152
+ "product_type",
153
+ "band",
154
+ "cdr_type",
155
+ "data_format",
156
+ "dataset_type",
157
+ "day",
158
+ "download_format",
159
+ "ensemble_member",
160
+ "experiment",
161
+ "forcing_type",
162
+ "gcm",
163
+ "hday",
164
+ "hmonth",
165
+ "horizontal_resolution",
166
+ "hydrological_model",
167
+ "hydrological_year",
168
+ "hyear",
169
+ "input_observations",
170
+ "leadtime_hour",
171
+ "leadtime_month",
172
+ "level",
173
+ "location",
174
+ "model",
175
+ "model_level",
176
+ "model_levels",
177
+ "month",
178
+ "nominal_day",
179
+ "originating_centre",
180
+ "period",
181
+ "pressure_level",
182
+ "processing_level",
183
+ "processing_type",
184
+ "product_version",
185
+ "quantity",
186
+ "rcm",
187
+ "region",
188
+ "release_version",
189
+ "satellite",
190
+ "sensor",
191
+ "sensor_and_algorithm",
192
+ "soil_level",
193
+ "sky_type",
194
+ "statistic",
195
+ "system_version",
196
+ "temporal_aggregation",
197
+ "time_aggregation",
198
+ "time_reference",
199
+ "time_step",
200
+ "variable",
201
+ "variable_type",
202
+ "version",
203
+ "year",
204
+ ]
205
+
206
+
207
+ def keywords_to_mdt(
208
+ keywords: List[str], prefix: Optional[str] = None
209
+ ) -> Dict[str, Any]:
94
210
  """
211
+ Make metadata mapping dict from a list of keywords
95
212
 
96
- def count_hits(
97
- self, count_url: Optional[str] = None, result_type: Optional[str] = None
98
- ) -> int:
99
- """Count method that will always return 1."""
100
- return 1
213
+ prefix:keyword:
214
+ - keyword
215
+ - $."prefix:keyword"
101
216
 
102
- def collect_search_urls(
103
- self,
104
- prep: PreparedSearch = PreparedSearch(),
105
- **kwargs: Any,
106
- ) -> Tuple[List[str], int]:
107
- """Wraps PostJsonSearch.collect_search_urls to force product count to 1"""
108
- urls, _ = super(BuildPostSearchResult, self).collect_search_urls(prep, **kwargs)
109
- return urls, 1
217
+ >>> keywords_to_mdt(["month", "year"])
218
+ {'month': ['month', '$."month"'], 'year': ['year', '$."year"']}
219
+ >>> keywords_to_mdt(["month", "year"], "ecmwf")
220
+ {'ecmwf:month': ['month', '$."ecmwf:month"'], 'ecmwf:year': ['year', '$."ecmwf:year"']}
110
221
 
111
- def do_search(
112
- self, prep: PreparedSearch = PreparedSearch(items_per_page=None), **kwargs: Any
113
- ) -> List[Dict[str, Any]]:
114
- """Perform the actual search request, and return result in a single element."""
115
- prep.url = prep.search_urls[0]
116
- prep.info_message = f"Sending search request: {prep.url}"
117
- prep.exception_message = (
118
- f"Skipping error while searching for {self.provider}"
119
- f" {self.__class__.__name__} instance"
120
- )
121
- response = self._request(prep)
122
-
123
- return [response.json()]
222
+ :param keywords: List of keywords to be converted
223
+ :param prefix: prefix to be added to the parameter in the mapping
224
+ :return: metadata mapping dict
225
+ """
226
+ mdt: Dict[str, Any] = {}
227
+ for keyword in keywords:
228
+ key = f"{prefix}:{keyword}" if prefix else keyword
229
+ mdt[key] = [keyword, f'$."{key}"']
230
+ return mdt
124
231
 
125
- def normalize_results(
126
- self, results: RawSearchResult, **kwargs: Any
127
- ) -> List[EOProduct]:
128
- """Build :class:`~eodag.api.product._product.EOProduct` from provider result
129
232
 
130
- :param results: Raw provider result as single dict in list
131
- :param kwargs: Search arguments
132
- :returns: list of single :class:`~eodag.api.product._product.EOProduct`
133
- """
134
- product_type = kwargs.get("productType")
233
+ def strip_quotes(value: Any) -> Any:
234
+ """Strip superfluous quotes from elements (added by mapping converter to_geojson).
135
235
 
136
- result = results[0]
236
+ >>> strip_quotes("'abc'")
237
+ 'abc'
238
+ >>> strip_quotes(["'abc'", '"def'])
239
+ ['abc', 'def']
137
240
 
138
- # datacube query string got from previous search
139
- _dc_qs = kwargs.pop("_dc_qs", None)
140
- if _dc_qs is not None:
141
- qs = unquote_plus(unquote_plus(_dc_qs))
142
- sorted_unpaginated_query_params = geojson.loads(qs)
143
- else:
144
- # update result with query parameters without pagination (or search-only params)
145
- if isinstance(
146
- self.config.pagination["next_page_query_obj"], str
147
- ) and hasattr(results, "query_params_unpaginated"):
148
- unpaginated_query_params = results.query_params_unpaginated
149
- elif isinstance(self.config.pagination["next_page_query_obj"], str):
150
- next_page_query_obj = orjson.loads(
151
- self.config.pagination["next_page_query_obj"].format()
152
- )
153
- unpaginated_query_params = {
154
- k: v[0] if (isinstance(v, list) and len(v) == 1) else v
155
- for k, v in results.query_params.items()
156
- if (k, v) not in next_page_query_obj.items()
157
- }
158
- else:
159
- unpaginated_query_params = self.query_params
160
-
161
- # query hash, will be used to build a product id
162
- sorted_unpaginated_query_params = dict_items_recursive_sort(
163
- unpaginated_query_params
164
- )
165
-
166
- # use all available query_params to parse properties
167
- result = dict(
168
- result,
169
- **sorted_unpaginated_query_params,
170
- qs=sorted_unpaginated_query_params,
241
+ :param value: value from which quotes should be removed (should be either str or list)
242
+ :return: value without quotes
243
+ :raises: NotImplementedError
244
+ """
245
+ if isinstance(value, (list, tuple)):
246
+ return [strip_quotes(v) for v in value]
247
+ elif isinstance(value, dict):
248
+ raise NotImplementedError("Dict value is not supported.")
249
+ else:
250
+ return str(value).strip("'\"")
251
+
252
+
253
+ def _update_properties_from_element(
254
+ prop: Dict[str, Any], element: Dict[str, Any], values: List[str]
255
+ ) -> None:
256
+ """updates a property dict with the given values based on the information from the element dict
257
+ e.g. the type is set based on the type of the element
258
+ """
259
+ # multichoice elements are transformed into array
260
+ if element["type"] in ("StringListWidget", "StringListArrayWidget"):
261
+ prop["type"] = "array"
262
+ if values:
263
+ prop["items"] = {"type": "string", "enum": sorted(values)}
264
+
265
+ # single choice elements are transformed into string
266
+ elif element["type"] in (
267
+ "StringChoiceWidget",
268
+ "DateRangeWidget",
269
+ "FreeformInputWidget",
270
+ ):
271
+ prop["type"] = "string"
272
+ if values:
273
+ prop["enum"] = sorted(values)
274
+
275
+ # a bbox element
276
+ elif element["type"] in ["GeographicExtentWidget", "GeographicExtentMapWidget"]:
277
+ prop.update(
278
+ {
279
+ "type": "array",
280
+ "minItems": 4,
281
+ "additionalItems": False,
282
+ "items": [
283
+ {
284
+ "type": "number",
285
+ "maximum": 180,
286
+ "minimum": -180,
287
+ "description": "West border of the bounding box",
288
+ },
289
+ {
290
+ "type": "number",
291
+ "maximum": 90,
292
+ "minimum": -90,
293
+ "description": "South border of the bounding box",
294
+ },
295
+ {
296
+ "type": "number",
297
+ "maximum": 180,
298
+ "minimum": -180,
299
+ "description": "East border of the bounding box",
300
+ },
301
+ {
302
+ "type": "number",
303
+ "maximum": 90,
304
+ "minimum": -90,
305
+ "description": "North border of the bounding box",
306
+ },
307
+ ],
308
+ }
171
309
  )
172
310
 
173
- # remove unwanted query params
174
- for param in getattr(self.config, "remove_from_query", []):
175
- sorted_unpaginated_query_params.pop(param, None)
176
-
177
- qs = geojson.dumps(sorted_unpaginated_query_params)
178
-
179
- query_hash = hashlib.sha1(str(qs).encode("UTF-8")).hexdigest()
311
+ # DateRangeWidget is a calendar date picker
312
+ if element["type"] == "DateRangeWidget":
313
+ prop["description"] = "date formatted like yyyy-mm-dd/yyyy-mm-dd"
180
314
 
181
- # update result with product_type_def_params and search args if not None (and not auth)
182
- kwargs.pop("auth", None)
183
- result.update(results.product_type_def_params)
184
- result = dict(result, **{k: v for k, v in kwargs.items() if v is not None})
315
+ if description := element.get("help"):
316
+ prop["description"] = description
185
317
 
186
- # parse properties
187
- parsed_properties = properties_from_json(
188
- result,
189
- self.config.metadata_mapping,
190
- discovery_config=getattr(self.config, "discover_metadata", {}),
191
- )
192
318
 
193
- if not product_type:
194
- product_type = parsed_properties.get("productType", None)
319
+ def ecmwf_format(v: str) -> str:
320
+ """Add ECMWF prefix to value v if v is a ECMWF keyword."""
321
+ return "ecmwf:" + v if v in ECMWF_KEYWORDS + COP_DS_KEYWORDS else v
195
322
 
196
- # build product id
197
- id_prefix = (product_type or self.provider).upper()
198
- product_id = "%s_%s_%s_%s" % (
199
- id_prefix,
200
- parsed_properties["startTimeFromAscendingNode"]
201
- .split("T")[0]
202
- .replace("-", ""),
203
- parsed_properties["completionTimeFromAscendingNode"]
204
- .split("T")[0]
205
- .replace("-", ""),
206
- query_hash,
207
- )
208
- parsed_properties["id"] = parsed_properties["title"] = product_id
209
323
 
210
- # update downloadLink and orderLink
211
- parsed_properties["_dc_qs"] = quote_plus(qs)
212
- if parsed_properties["downloadLink"] != "Not Available":
213
- parsed_properties["downloadLink"] += f"?{qs}"
324
+ class ECMWFSearch(PostJsonSearch):
325
+ """ECMWF search plugin.
214
326
 
215
- # parse metadata needing downloadLink
216
- dl_path = Fields("downloadLink")
217
- dl_path_from_root = Child(Root(), dl_path)
218
- for param, mapping in self.config.metadata_mapping.items():
219
- if dl_path in mapping or dl_path_from_root in mapping:
220
- parsed_properties.update(
221
- properties_from_json(parsed_properties, {param: mapping})
222
- )
223
-
224
- # use product_type_config as default properties
225
- parsed_properties = dict(
226
- getattr(self.config, "product_type_config", {}),
227
- **parsed_properties,
228
- )
229
-
230
- product = EOProduct(
231
- provider=self.provider,
232
- productType=product_type,
233
- properties=parsed_properties,
234
- )
235
-
236
- return [
237
- product,
238
- ]
239
-
240
-
241
- class BuildSearchResult(BuildPostSearchResult):
242
- """BuildSearchResult search plugin.
243
-
244
- This plugin builds a single :class:`~eodag.api.search_result.SearchResult` object
327
+ This plugin builds a :class:`~eodag.api.search_result.SearchResult` containing a single product
245
328
  using given query parameters as product properties.
246
329
 
247
- The available configuration parameters inherits from parent classes
248
- (:class:`~eodag.plugins.search.build_search_result.BuildPostSearchResult`,
249
- :class:`~eodag.plugins.search.qssearch.PostJsonSearch` and
250
- :class:`~eodag.plugins.search.qssearch.QueryStringSearch`), with particularly for this plugin:
330
+ The available configuration parameters inherits from parent classes, with some particular parameters
331
+ for this plugin.
251
332
 
252
- :param provider: provider name
333
+ :param provider: An eodag providers configuration dictionary
253
334
  :param config: Search plugin configuration:
254
335
 
255
- * :attr:`~eodag.config.PluginConfig.end_date_excluded` (``bool``): Set to ``False`` if provider
256
- does not include end date in the search request; In this case, if the end date is at midnight,
257
- the previous day will be used. default: ``True``
258
-
336
+ * :attr:`~eodag.config.PluginConfig.remove_from_query` (``List[str]``): List of parameters
337
+ used to parse metadata but that must not be included to the query
338
+ * :attr:`~eodag.config.PluginConfig.end_date_excluded` (``bool``): Set to `False` if
339
+ provider does not include end date to search
340
+ * :attr:`~eodag.config.PluginConfig.discover_queryables`
341
+ (:class:`~eodag.config.PluginConfig.DiscoverQueryables`): configuration to fetch the queryables from a
342
+ provider queryables endpoint; It has the following keys:
343
+
344
+ * :attr:`~eodag.config.PluginConfig.DiscoverQueryables.fetch_url` (``str``): url to fetch the queryables valid
345
+ for all product types
346
+ * :attr:`~eodag.config.PluginConfig.DiscoverQueryables.product_type_fetch_url` (``str``): url to fetch the
347
+ queryables for a specific product type
348
+ * :attr:`~eodag.config.PluginConfig.DiscoverQueryables.constraints_url` (``str``): url of the constraint file
349
+ used to build queryables
259
350
  """
260
351
 
261
352
  def __init__(self, provider: str, config: PluginConfig) -> None:
262
- # init self.config.metadata_mapping using Search Base plugin
263
- Search.__init__(self, provider, config)
353
+ # cache fetching method
354
+ self.fetch_data = functools.lru_cache()(self._fetch_data)
355
+
356
+ config.metadata_mapping = {
357
+ **keywords_to_mdt(ECMWF_KEYWORDS + COP_DS_KEYWORDS, "ecmwf"),
358
+ **config.metadata_mapping,
359
+ }
360
+
361
+ super().__init__(provider, config)
264
362
 
265
363
  self.config.__dict__.setdefault("api_endpoint", "")
266
364
 
267
365
  # needed by QueryStringSearch.build_query_string / format_free_text_search
268
366
  self.config.__dict__.setdefault("free_text_search_operations", {})
269
367
  # needed for compatibility
270
- self.config.__dict__.setdefault("pagination", {"next_page_query_obj": "{{}}"})
368
+ self.config.pagination.setdefault("next_page_query_obj", "{{}}")
271
369
 
272
370
  # parse jsonpath on init: product type specific metadata-mapping
273
371
  for product_type in self.config.products.keys():
@@ -305,7 +403,13 @@ class BuildSearchResult(BuildPostSearchResult):
305
403
  ] = product_type_metadata_mapping
306
404
 
307
405
  def do_search(self, *args: Any, **kwargs: Any) -> List[Dict[str, Any]]:
308
- """Should perform the actual search request."""
406
+ """Should perform the actual search request.
407
+
408
+ :param args: arguments to be used in the search
409
+ :param kwargs: keyword arguments to be used in the search
410
+ :return: list containing the results from the provider in json format
411
+ """
412
+ # no real search. We fake it all
309
413
  return [{}]
310
414
 
311
415
  def query(
@@ -313,37 +417,55 @@ class BuildSearchResult(BuildPostSearchResult):
313
417
  prep: PreparedSearch = PreparedSearch(),
314
418
  **kwargs: Any,
315
419
  ) -> Tuple[List[EOProduct], Optional[int]]:
316
- """Build ready-to-download SearchResult"""
420
+ """Build ready-to-download SearchResult
317
421
 
318
- self._preprocess_search_params(kwargs)
422
+ :param prep: :class:`~eodag.plugins.search.PreparedSearch` object containing information needed for the search
423
+ :param kwargs: keyword arguments to be used in the search
424
+ :returns: list of products and number of products (optional)
425
+ """
426
+ product_type = prep.product_type
427
+ if not product_type:
428
+ product_type = kwargs.get("productType", None)
429
+ self._preprocess_search_params(kwargs, product_type)
430
+ result, num_items = super().query(prep, **kwargs)
431
+ if prep.count and not num_items:
432
+ num_items = 1
319
433
 
320
- return BuildPostSearchResult.query(self, prep, **kwargs)
434
+ return result, num_items
321
435
 
322
436
  def clear(self) -> None:
323
437
  """Clear search context"""
324
- pass
438
+ super().clear()
325
439
 
326
440
  def build_query_string(
327
441
  self, product_type: str, **kwargs: Any
328
442
  ) -> Tuple[Dict[str, Any], str]:
329
- """Build The query string using the search parameters"""
443
+ """Build The query string using the search parameters
444
+
445
+ :param product_type: product type id
446
+ :param kwargs: keyword arguments to be used in the query string
447
+ :return: formatted query params and encode query string
448
+ """
330
449
  # parse kwargs as properties as they might be needed to build the query
331
450
  parsed_properties = properties_from_json(
332
451
  kwargs,
333
452
  self.config.metadata_mapping,
334
453
  )
335
454
  available_properties = {
336
- k: v
455
+ # We strip values of superfluous quotes (added by mapping converter to_geojson).
456
+ k: strip_quotes(v)
337
457
  for k, v in parsed_properties.items()
338
458
  if v not in [NOT_AVAILABLE, NOT_MAPPED]
339
459
  }
340
460
 
341
461
  # build and return the query
342
- return BuildPostSearchResult.build_query_string(
343
- self, product_type=product_type, **available_properties
462
+ return super().build_query_string(
463
+ product_type=product_type, **available_properties
344
464
  )
345
465
 
346
- def _preprocess_search_params(self, params: Dict[str, Any]) -> None:
466
+ def _preprocess_search_params(
467
+ self, params: Dict[str, Any], product_type: Optional[str]
468
+ ) -> None:
347
469
  """Preprocess search parameters before making a request to the CDS API.
348
470
 
349
471
  This method is responsible for checking and updating the provided search parameters
@@ -352,6 +474,7 @@ class BuildSearchResult(BuildPostSearchResult):
352
474
  in the input parameters, default values or values from the configuration are used.
353
475
 
354
476
  :param params: Search parameters to be preprocessed.
477
+ :param product_type: (optional) product type id
355
478
  """
356
479
  _dc_qs = params.get("_dc_qs", None)
357
480
  if _dc_qs is not None:
@@ -378,63 +501,43 @@ class BuildSearchResult(BuildPostSearchResult):
378
501
  non_none_params = {k: v for k, v in params.items() if v}
379
502
 
380
503
  # productType
381
- dataset = params.get("dataset", None)
504
+ dataset = params.get("ecmwf:dataset", None)
382
505
  params["productType"] = non_none_params.get("productType", dataset)
383
506
 
384
507
  # dates
385
- mission_start_dt = datetime.fromisoformat(
386
- self.get_product_type_cfg_value(
387
- "missionStartDate", DEFAULT_MISSION_START_DATE
388
- ).replace(
389
- "Z", "+00:00"
390
- ) # before 3.11
391
- )
392
-
393
- default_end_from_cfg = self.config.products.get(params["productType"], {}).get(
394
- "_default_end_date", None
395
- )
396
- default_end_str = (
397
- default_end_from_cfg
398
- or (
399
- datetime.now(timezone.utc)
400
- if params.get("startTimeFromAscendingNode")
401
- else mission_start_dt + timedelta(days=1)
402
- ).isoformat()
403
- )
404
-
405
- params["startTimeFromAscendingNode"] = non_none_params.get(
406
- "startTimeFromAscendingNode", mission_start_dt.isoformat()
407
- )
408
- params["completionTimeFromAscendingNode"] = non_none_params.get(
409
- "completionTimeFromAscendingNode", default_end_str
410
- )
508
+ # check if default dates have to be added
509
+ if getattr(self.config, "dates_required", False):
510
+ self._check_date_params(params, product_type)
411
511
 
412
512
  # adapt end date if it is midnight
413
- end_date_excluded = getattr(self.config, "end_date_excluded", True)
414
- is_datetime = True
415
- try:
416
- end_date = datetime.strptime(
417
- params["completionTimeFromAscendingNode"], "%Y-%m-%dT%H:%M:%SZ"
418
- )
419
- end_date = end_date.replace(tzinfo=tzutc())
420
- except ValueError:
513
+ if "completionTimeFromAscendingNode" in params:
514
+ end_date_excluded = getattr(self.config, "end_date_excluded", True)
515
+ is_datetime = True
421
516
  try:
422
517
  end_date = datetime.strptime(
423
- params["completionTimeFromAscendingNode"], "%Y-%m-%dT%H:%M:%S.%fZ"
518
+ params["completionTimeFromAscendingNode"], "%Y-%m-%dT%H:%M:%SZ"
424
519
  )
425
520
  end_date = end_date.replace(tzinfo=tzutc())
426
521
  except ValueError:
427
- end_date = isoparse(params["completionTimeFromAscendingNode"])
428
- is_datetime = False
429
- start_date = isoparse(params["startTimeFromAscendingNode"])
430
- if (
431
- not end_date_excluded
432
- and is_datetime
433
- and end_date > start_date
434
- and end_date == end_date.replace(hour=0, minute=0, second=0, microsecond=0)
435
- ):
436
- end_date += timedelta(days=-1)
437
- params["completionTimeFromAscendingNode"] = end_date.isoformat()
522
+ try:
523
+ end_date = datetime.strptime(
524
+ params["completionTimeFromAscendingNode"],
525
+ "%Y-%m-%dT%H:%M:%S.%fZ",
526
+ )
527
+ end_date = end_date.replace(tzinfo=tzutc())
528
+ except ValueError:
529
+ end_date = isoparse(params["completionTimeFromAscendingNode"])
530
+ is_datetime = False
531
+ start_date = isoparse(params["startTimeFromAscendingNode"])
532
+ if (
533
+ not end_date_excluded
534
+ and is_datetime
535
+ and end_date > start_date
536
+ and end_date
537
+ == end_date.replace(hour=0, minute=0, second=0, microsecond=0)
538
+ ):
539
+ end_date += timedelta(days=-1)
540
+ params["completionTimeFromAscendingNode"] = end_date.isoformat()
438
541
 
439
542
  # geometry
440
543
  if "geometry" in params:
@@ -449,94 +552,728 @@ class BuildSearchResult(BuildPostSearchResult):
449
552
  arguments)
450
553
  :returns: fetched queryable parameters dict
451
554
  """
452
- constraints_file_url = getattr(self.config, "constraints_file_url", "")
453
- if not constraints_file_url:
454
- return {}
455
- product_type = kwargs.pop("productType", None)
456
- if not product_type:
457
- return {}
555
+ product_type = kwargs.pop("productType")
556
+ product_type_config = self.config.products.get(product_type, {})
557
+ provider_product_type = (
558
+ product_type_config.get("ecmwf:dataset", None)
559
+ or product_type_config["productType"]
560
+ )
561
+ if "start" in kwargs:
562
+ kwargs["startTimeFromAscendingNode"] = kwargs.pop("start")
563
+ if "end" in kwargs:
564
+ kwargs["completionTimeFromAscendingNode"] = kwargs.pop("end")
565
+
566
+ # extract default datetime
567
+ processed_kwargs = deepcopy(kwargs)
568
+ self._preprocess_search_params(processed_kwargs, product_type)
569
+
570
+ constraints_url = format_metadata(
571
+ getattr(self.config, "discover_queryables", {}).get("constraints_url", ""),
572
+ **kwargs,
573
+ )
574
+ constraints: List[Dict[str, Any]] = self.fetch_data(constraints_url)
575
+
576
+ form_url = format_metadata(
577
+ getattr(self.config, "discover_queryables", {}).get("form_url", ""),
578
+ **kwargs,
579
+ )
580
+ form = self.fetch_data(form_url)
458
581
 
459
- provider_product_type = self.config.products.get(product_type, {}).get(
460
- "dataset", None
582
+ formated_kwargs = self.format_as_provider_keyword(
583
+ product_type, processed_kwargs
461
584
  )
462
- user_provider_product_type = kwargs.pop("dataset", None)
585
+ # we re-apply kwargs input to consider override of year, month, day and time.
586
+ for key in kwargs:
587
+ if key.startswith("ecmwf:"):
588
+ formated_kwargs[key.replace("ecmwf:", "")] = kwargs[key]
589
+ elif key in (
590
+ "startTimeFromAscendingNode",
591
+ "completionTimeFromAscendingNode",
592
+ "geom",
593
+ ):
594
+ formated_kwargs[key] = kwargs[key]
595
+ else:
596
+ raise ValidationError(
597
+ f"{key} is not a queryable parameter for {self.provider}"
598
+ )
599
+
600
+ # we use non empty kwargs as default to integrate user inputs
601
+ # it is needed because pydantic json schema does not represent "value"
602
+ # but only "default"
603
+ non_empty_formated: Dict[str, Any] = {
604
+ k: v
605
+ for k, v in formated_kwargs.items()
606
+ if v and (not isinstance(v, list) or all(v))
607
+ }
608
+ non_empty_kwargs: Dict[str, Any] = {
609
+ k: v
610
+ for k, v in processed_kwargs.items()
611
+ if v and (not isinstance(v, list) or all(v))
612
+ }
613
+
614
+ required_keywords: Set[str] = set()
615
+
616
+ # calculate available values
617
+ if constraints:
618
+ # Apply constraints filtering
619
+ available_values = self.available_values_from_constraints(
620
+ constraints,
621
+ non_empty_formated,
622
+ form_keywords=[f["name"] for f in form],
623
+ )
624
+
625
+ # Pre-compute the required keywords (present in all constraint dicts)
626
+ # when form, required keywords are extracted directly from form
627
+ if not form:
628
+ required_keywords = set(constraints[0].keys())
629
+ for constraint in constraints[1:]:
630
+ required_keywords.intersection_update(constraint.keys())
631
+ else:
632
+ values_url = getattr(self.config, "available_values_url", "")
633
+ if not values_url:
634
+ return self.queryables_from_metadata_mapping(product_type)
635
+ if "{" in values_url:
636
+ values_url = values_url.format(productType=provider_product_type)
637
+ data = self.fetch_data(values_url)
638
+ available_values = data["constraints"]
639
+ required_keywords = data.get("required", [])
640
+
641
+ # To check if all keywords are queryable parameters, we check if they are in the
642
+ # available values or the product type config (available values calculated from the
643
+ # constraints might not include all queryables)
644
+ for keyword in kwargs:
645
+ if (
646
+ keyword
647
+ not in available_values.keys()
648
+ | product_type_config.keys()
649
+ | {
650
+ "startTimeFromAscendingNode",
651
+ "completionTimeFromAscendingNode",
652
+ "geom",
653
+ }
654
+ and keyword.replace("ecmwf:", "") not in available_values
655
+ ):
656
+ raise ValidationError(f"{keyword} is not a queryable parameter")
657
+
658
+ # generate queryables
659
+ if form:
660
+ queryables = self.queryables_by_form(
661
+ form,
662
+ available_values,
663
+ non_empty_formated,
664
+ )
665
+ else:
666
+ queryables = self.queryables_by_values(
667
+ available_values, list(required_keywords), non_empty_kwargs
668
+ )
669
+
670
+ # ecmwf:date is replaced by start and end.
671
+ # start and end filters are supported whenever combinations of "year", "month", "day" filters exist
463
672
  if (
464
- user_provider_product_type
465
- and user_provider_product_type != provider_product_type
673
+ queryables.pop("ecmwf:date", None)
674
+ or "ecmwf:year" in queryables
675
+ or "ecmwf:hyear" in queryables
466
676
  ):
467
- raise ValidationError(
468
- f"Cannot change dataset from {provider_product_type} to {user_provider_product_type}"
677
+ queryables.update(
678
+ {
679
+ "start": Queryables.get_with_default(
680
+ "start", non_empty_kwargs.get("startTimeFromAscendingNode")
681
+ ),
682
+ "end": Queryables.get_with_default(
683
+ "end",
684
+ non_empty_kwargs.get("completionTimeFromAscendingNode"),
685
+ ),
686
+ }
469
687
  )
470
688
 
471
- # defaults
472
- default_queryables = self._get_defaults_as_queryables(product_type)
473
- # remove dataset from queryables
474
- default_queryables.pop("dataset", None)
689
+ # area is geom in EODAG.
690
+ if queryables.pop("area", None):
691
+ queryables["geom"] = Annotated[
692
+ Union[str, Dict[str, float], BaseGeometry],
693
+ Field(
694
+ None,
695
+ description="Read EODAG documentation for all supported geometry format.",
696
+ ),
697
+ ]
475
698
 
476
- non_empty_kwargs = {k: v for k, v in kwargs.items() if v}
699
+ return queryables
477
700
 
478
- if "{" in constraints_file_url:
479
- constraints_file_url = constraints_file_url.format(
480
- dataset=provider_product_type
481
- )
482
- constraints = fetch_constraints(constraints_file_url, self)
483
- if not constraints:
484
- return default_queryables
485
-
486
- constraint_params: Dict[str, Dict[str, Set[Any]]] = {}
487
- if len(kwargs) == 0:
488
- # get values from constraints without additional filters
489
- for constraint in constraints:
490
- for key in constraint.keys():
491
- if key in constraint_params:
492
- constraint_params[key]["enum"].update(constraint[key])
493
- else:
494
- constraint_params[key] = {}
495
- constraint_params[key]["enum"] = set(constraint[key])
496
- else:
497
- # get values from constraints with additional filters
498
- constraints_input_params = {k: v for k, v in non_empty_kwargs.items()}
499
- constraint_params = get_constraint_queryables_with_additional_params(
500
- constraints, constraints_input_params, self, product_type
701
+ def available_values_from_constraints(
702
+ self,
703
+ constraints: list[Dict[str, Any]],
704
+ input_keywords: Dict[str, Any],
705
+ form_keywords: List[str],
706
+ ) -> Dict[str, List[str]]:
707
+ """
708
+ Filter constraints using input_keywords. Return list of available queryables.
709
+ All constraint entries must have the same parameters.
710
+
711
+ :param constraints: list of constraints received from the provider
712
+ :param input_keywords: dict of input parameters given by the user
713
+ :param form_keywords: list of keyword names from the provider form endpoint
714
+ :return: dict with available values for each parameter
715
+ """
716
+ # get ordered constraint keywords
717
+ constraints_keywords = list(
718
+ OrderedDict.fromkeys(k for c in constraints for k in c.keys())
719
+ )
720
+
721
+ # prepare ordered input keywords formatted as provider's keywords
722
+ # required to filter with constraints
723
+ ordered_keywords = (
724
+ [kw for kw in form_keywords if kw in constraints_keywords]
725
+ if form_keywords
726
+ else constraints_keywords
727
+ )
728
+
729
+ # filter constraint entries matching input keyword values
730
+ filtered_constraints: List[Dict[str, Any]]
731
+
732
+ parsed_keywords: List[str] = []
733
+ for keyword in ordered_keywords:
734
+ values = input_keywords.get(keyword)
735
+
736
+ if values is None:
737
+ parsed_keywords.append(keyword)
738
+ continue
739
+
740
+ # we only compare list of strings.
741
+ if isinstance(values, dict):
742
+ raise ValidationError(
743
+ f"Parameter value as object is not supported: {keyword}={values}"
744
+ )
745
+ filter_v = values if isinstance(values, (list, tuple)) else [values]
746
+
747
+ # We convert every single value to a list of string
748
+ # We strip values of superfluous quotes (added by mapping converter to_geojson).
749
+ # ECMWF accept values with /to/. We need to split it to an array
750
+ # ECMWF accept values in format val1/val2. We need to split it to an array
751
+ sep = re.compile(r"/to/|/")
752
+ filter_v = [i for v in filter_v for i in sep.split(strip_quotes(v))]
753
+
754
+ # special handling for time 0000 converted to 0 by pre-formating with metadata_mapping
755
+ if keyword.split(":")[-1] == "time":
756
+ filter_v = ["0000" if str(v) == "0" else v for v in filter_v]
757
+
758
+ # Collect missing values to report errors
759
+ missing_values = set(filter_v)
760
+
761
+ # Filter constraints and check for missing values
762
+ filtered_constraints = []
763
+ for entry in constraints:
764
+ # Filter based on the presence of any value in filter_v
765
+ entry_values = entry.get(keyword, [])
766
+
767
+ # date constraint may be intervals. We identify intervals with a "/" in the value
768
+ # we assume that if the first value is an interval, all values are intervals
769
+ present_values = []
770
+ if keyword == "date" and "/" in entry[keyword][0]:
771
+ if any(is_range_in_range(x, values[0]) for x in entry[keyword]):
772
+ present_values = filter_v
773
+ else:
774
+ present_values = [
775
+ value for value in filter_v if value in entry_values
776
+ ]
777
+
778
+ # Remove present values from the missing_values set
779
+ missing_values -= set(present_values)
780
+
781
+ if present_values:
782
+ filtered_constraints.append(entry)
783
+
784
+ # raise an error as no constraint entry matched the input keywords
785
+ # raise an error if one value from input is not allowed
786
+ if not filtered_constraints or missing_values:
787
+ allowed_values = list(
788
+ {value for c in constraints for value in c.get(keyword, [])}
789
+ )
790
+ # restore ecmwf: prefix before raising error
791
+ keyword = f"ecmwf:{keyword}"
792
+
793
+ all_keywords_str = ""
794
+ if len(parsed_keywords) > 1:
795
+ keywords = [
796
+ f"ecmwf:{k}={pk}"
797
+ for k in parsed_keywords
798
+ if (pk := input_keywords.get(k))
799
+ ]
800
+ all_keywords_str = f" with {', '.join(keywords)}"
801
+
802
+ raise ValidationError(
803
+ f"{keyword}={values} is not available"
804
+ f"{all_keywords_str}."
805
+ f" Allowed values are {', '.join(allowed_values)}."
806
+ )
807
+
808
+ parsed_keywords.append(keyword)
809
+ constraints = filtered_constraints
810
+
811
+ available_values: Dict[str, Any] = {k: set() for k in ordered_keywords}
812
+
813
+ # we aggregate the constraint entries left
814
+ for entry in constraints:
815
+ for key, value in entry.items():
816
+ available_values[key].update(value)
817
+
818
+ return {k: list(v) for k, v in available_values.items()}
819
+
820
+ def queryables_by_form(
821
+ self,
822
+ form: List[Dict[str, Any]],
823
+ available_values: Dict[str, List[str]],
824
+ defaults: Dict[str, Any],
825
+ ) -> Dict[str, Annotated[Any, FieldInfo]]:
826
+ """
827
+ Generate Annotated field definitions from form entries and available values
828
+ Used by Copernicus services like cop_cds, cop_ads, cop_ewds.
829
+
830
+ :param form: data fetched from the form endpoint of the provider
831
+ :param available_values: available values for each parameter
832
+ :param defaults: default values for the parameters
833
+ :return: dict of annotated queryables
834
+ """
835
+ queryables: Dict[str, Annotated[Any, FieldInfo]] = {}
836
+
837
+ required_list: List[str] = []
838
+ for element in form:
839
+ name: str = element["name"]
840
+
841
+ # those are not parameter elements.
842
+ if name in ("area_group", "global", "warning", "licences"):
843
+ continue
844
+ if "type" not in element or element["type"] == "FreeEditionWidget":
845
+ continue
846
+
847
+ # ordering done by id -> set id to high value if not present -> element will be last
848
+ if "id" not in element:
849
+ element["id"] = 100
850
+
851
+ prop = {"title": element.get("label", name)}
852
+
853
+ details = element.get("details", {})
854
+
855
+ # add values from form if keyword was not in constraints
856
+ values = (
857
+ available_values[name]
858
+ if name in available_values
859
+ else details.get("values")
501
860
  )
502
- # query params that are not in constraints but might be default queryables
503
- if len(constraint_params) == 1 and "not_available" in constraint_params:
504
- not_queryables: Set[str] = set()
505
- for constraint_param in constraint_params["not_available"]["enum"]:
506
- param = CommonQueryables.get_queryable_from_alias(constraint_param)
507
- if param in dict(
508
- CommonQueryables.model_fields, **default_queryables
509
- ):
510
- non_empty_kwargs.pop(constraint_param)
511
- else:
512
- not_queryables.add(constraint_param)
513
- if not_queryables:
514
- raise ValidationError(
515
- f"parameter(s) {not_queryables} not queryable"
861
+
862
+ # updates the properties with the values given based on the information from the element
863
+ _update_properties_from_element(prop, element, values)
864
+
865
+ default = defaults.get(name)
866
+
867
+ if details:
868
+ fields = details.get("fields")
869
+ if fields and (comment := fields[0].get("comment")):
870
+ prop["description"] = comment
871
+
872
+ if d := details.get("default"):
873
+ default = default or (d[0] if fields else d)
874
+
875
+ if name == "area" and isinstance(default, dict):
876
+ default = list(default.values())
877
+
878
+ if default:
879
+ # We strip values of superfluous quotes (addded by mapping converter to_geojson).
880
+ default = strip_quotes(default)
881
+
882
+ # sometimes form returns default as array instead of string
883
+ if default and prop["type"] == "string" and isinstance(default, list):
884
+ default = ",".join(default)
885
+
886
+ is_required = bool(element.get("required"))
887
+ if is_required:
888
+ required_list.append(name)
889
+
890
+ queryables[ecmwf_format(name)] = Annotated[
891
+ get_args(
892
+ json_field_definition_to_python(
893
+ prop,
894
+ default_value=default,
895
+ required=is_required,
516
896
  )
517
- else:
518
- # get constraints again without common queryables
519
- constraint_params = (
520
- get_constraint_queryables_with_additional_params(
521
- constraints, non_empty_kwargs, self, product_type
522
- )
897
+ )
898
+ ]
899
+
900
+ return queryables
901
+
902
+ def queryables_by_values(
903
+ self,
904
+ available_values: Dict[str, List[str]],
905
+ required_keywords: List[str],
906
+ defaults: Dict[str, Any],
907
+ ) -> Dict[str, Annotated[Any, FieldInfo]]:
908
+ """
909
+ Generate Annotated field definitions from available values.
910
+ Used by ECMWF data providers like dedt_lumi.
911
+
912
+ :param available_values: available values for each parameter
913
+ :param required_keywords: list of required parameters
914
+ :param defaults: default values for the parameters
915
+ :return: dict of annotated queryables
916
+ """
917
+ # Rename keywords from form with metadata mapping.
918
+ # Needed to map constraints like "xxxx" to eodag parameter "ecmwf:xxxx"
919
+ required = [ecmwf_format(k) for k in required_keywords]
920
+
921
+ queryables: Dict[str, Annotated[Any, FieldInfo]] = {}
922
+ for name, values in available_values.items():
923
+ # Rename keywords from form with metadata mapping.
924
+ # Needed to map constraints like "xxxx" to eodag parameter "ecmwf:xxxx"
925
+ key = ecmwf_format(name)
926
+
927
+ default = defaults.get(key)
928
+
929
+ queryables[key] = Annotated[
930
+ get_args(
931
+ json_field_definition_to_python(
932
+ {"type": "string", "title": name, "enum": values},
933
+ default_value=strip_quotes(default) if default else None,
934
+ required=bool(key in required),
523
935
  )
936
+ )
937
+ ]
938
+
939
+ return queryables
940
+
941
+ def format_as_provider_keyword(
942
+ self, product_type: str, properties: Dict[str, Any]
943
+ ) -> Dict[str, Any]:
944
+ """Return provider equivalent keyword names from EODAG keywords.
945
+
946
+ :param product_type: product type id
947
+ :param properties: dict of properties to be formatted
948
+ :return: dict of formatted properties
949
+ """
950
+ parsed_properties = properties_from_json(
951
+ properties,
952
+ self.config.metadata_mapping,
953
+ )
954
+ available_properties = {
955
+ k: v
956
+ for k, v in parsed_properties.items()
957
+ if v not in [NOT_AVAILABLE, NOT_MAPPED]
958
+ }
959
+ return format_query_params(product_type, self.config, available_properties)
960
+
961
+ def _fetch_data(self, url: str) -> Any:
962
+ """
963
+ fetches from a provider elements like constraints or forms.
964
+
965
+ :param url: url from which the constraints can be fetched
966
+ :returns: json file content fetched from the provider
967
+ """
968
+ if not url:
969
+ return []
970
+
971
+ auth = (
972
+ self.auth
973
+ if hasattr(self, "auth") and isinstance(self.auth, AuthBase)
974
+ else None
975
+ )
976
+ timeout = getattr(self.config, "timeout", HTTP_REQ_TIMEOUT)
977
+ return fetch_json(url, auth=auth, timeout=timeout)
524
978
 
525
- field_definitions: Dict[str, Any] = {}
526
- for json_param, json_mtd in constraint_params.items():
527
- param = (
528
- get_queryable_from_provider(
529
- json_param, self.get_metadata_mapping(product_type)
979
+ def normalize_results(
980
+ self, results: RawSearchResult, **kwargs: Any
981
+ ) -> List[EOProduct]:
982
+ """Build :class:`~eodag.api.product._product.EOProduct` from provider result
983
+
984
+ :param results: Raw provider result as single dict in list
985
+ :param kwargs: Search arguments
986
+ :returns: list of single :class:`~eodag.api.product._product.EOProduct`
987
+ """
988
+
989
+ product_type = kwargs.get("productType")
990
+
991
+ result = results[0]
992
+
993
+ # datacube query string got from previous search
994
+ _dc_qs = kwargs.pop("_dc_qs", None)
995
+ if _dc_qs is not None:
996
+ qs = unquote_plus(unquote_plus(_dc_qs))
997
+ sorted_unpaginated_query_params = geojson.loads(qs)
998
+ else:
999
+ # update result with query parameters without pagination (or search-only params)
1000
+ if isinstance(
1001
+ self.config.pagination["next_page_query_obj"], str
1002
+ ) and hasattr(results, "query_params_unpaginated"):
1003
+ unpaginated_query_params = results.query_params_unpaginated
1004
+ elif isinstance(self.config.pagination["next_page_query_obj"], str):
1005
+ next_page_query_obj = orjson.loads(
1006
+ self.config.pagination["next_page_query_obj"].format()
530
1007
  )
531
- or json_param
1008
+ unpaginated_query_params = {
1009
+ k: v
1010
+ for k, v in results.query_params.items()
1011
+ if (k, v) not in next_page_query_obj.items()
1012
+ }
1013
+ else:
1014
+ unpaginated_query_params = self.query_params
1015
+ # query hash, will be used to build a product id
1016
+ sorted_unpaginated_query_params = dict_items_recursive_sort(
1017
+ unpaginated_query_params
1018
+ )
1019
+
1020
+ # use all available query_params to parse properties
1021
+ result = dict(
1022
+ result,
1023
+ **sorted_unpaginated_query_params,
1024
+ qs=sorted_unpaginated_query_params,
1025
+ )
1026
+
1027
+ # remove unwanted query params
1028
+ for param in getattr(self.config, "remove_from_query", []):
1029
+ sorted_unpaginated_query_params.pop(param, None)
1030
+
1031
+ qs = geojson.dumps(sorted_unpaginated_query_params)
1032
+
1033
+ query_hash = hashlib.sha1(str(qs).encode("UTF-8")).hexdigest()
1034
+
1035
+ # update result with product_type_def_params and search args if not None (and not auth)
1036
+ kwargs.pop("auth", None)
1037
+ result.update(results.product_type_def_params)
1038
+ result = dict(result, **{k: v for k, v in kwargs.items() if v is not None})
1039
+
1040
+ # parse properties
1041
+ parsed_properties = properties_from_json(
1042
+ result,
1043
+ self.config.metadata_mapping,
1044
+ discovery_config=getattr(self.config, "discover_metadata", {}),
1045
+ )
1046
+
1047
+ if not product_type:
1048
+ product_type = parsed_properties.get("productType", None)
1049
+
1050
+ # build product id
1051
+ id_prefix = (product_type or self.provider).upper()
1052
+ if (
1053
+ "startTimeFromAscendingNode" in parsed_properties
1054
+ and parsed_properties["startTimeFromAscendingNode"] != "Not Available"
1055
+ and "completionTimeFromAscendingNode" in parsed_properties
1056
+ and parsed_properties["completionTimeFromAscendingNode"] != "Not Available"
1057
+ ):
1058
+ product_id = "%s_%s_%s_%s" % (
1059
+ id_prefix,
1060
+ parsed_properties["startTimeFromAscendingNode"]
1061
+ .split("T")[0]
1062
+ .replace("-", ""),
1063
+ parsed_properties["completionTimeFromAscendingNode"]
1064
+ .split("T")[0]
1065
+ .replace("-", ""),
1066
+ query_hash,
532
1067
  )
533
- default = kwargs.get(param, None) or self.config.products.get(
534
- product_type, {}
535
- ).get(param, None)
536
- annotated_def = json_field_definition_to_python(
537
- json_mtd, default_value=default, required=True
1068
+ elif (
1069
+ "startTimeFromAscendingNode" in parsed_properties
1070
+ and parsed_properties["startTimeFromAscendingNode"] != "Not Available"
1071
+ ):
1072
+ product_id = "%s_%s_%s" % (
1073
+ id_prefix,
1074
+ parsed_properties["startTimeFromAscendingNode"]
1075
+ .split("T")[0]
1076
+ .replace("-", ""),
1077
+ query_hash,
538
1078
  )
539
- field_definitions[param] = get_args(annotated_def)
1079
+ else:
1080
+ product_id = f"{id_prefix}_{query_hash}"
1081
+
1082
+ parsed_properties["id"] = parsed_properties["title"] = product_id
1083
+
1084
+ # update downloadLink and orderLink
1085
+ parsed_properties["_dc_qs"] = quote_plus(qs)
1086
+ if parsed_properties["downloadLink"] != "Not Available":
1087
+ parsed_properties["downloadLink"] += f"?{qs}"
1088
+
1089
+ # parse metadata needing downloadLink
1090
+ dl_path = Fields("downloadLink")
1091
+ dl_path_from_root = Child(Root(), dl_path)
1092
+ for param, mapping in self.config.metadata_mapping.items():
1093
+ if dl_path in mapping or dl_path_from_root in mapping:
1094
+ parsed_properties.update(
1095
+ properties_from_json(parsed_properties, {param: mapping})
1096
+ )
1097
+
1098
+ # use product_type_config as default properties
1099
+ parsed_properties = dict(
1100
+ getattr(self.config, "product_type_config", {}),
1101
+ **parsed_properties,
1102
+ )
1103
+
1104
+ product = EOProduct(
1105
+ provider=self.provider,
1106
+ productType=product_type,
1107
+ properties=parsed_properties,
1108
+ )
1109
+
1110
+ return [
1111
+ product,
1112
+ ]
1113
+
1114
+ def count_hits(
1115
+ self, count_url: Optional[str] = None, result_type: Optional[str] = None
1116
+ ) -> int:
1117
+ """Count method that will always return 1.
1118
+
1119
+ :param count_url: not used, only here because this method overwrites count_hits from the parent class
1120
+ :param result_type: not used, only here because this method overwrites count_hits from the parent class
1121
+ :return: always 1
1122
+ """
1123
+ return 1
1124
+
1125
+
1126
+ class MeteoblueSearch(ECMWFSearch):
1127
+ """MeteoblueSearch search plugin.
1128
+
1129
+ This plugin, which inherits from :class:`~eodag.plugins.search.build_search_result.ECMWFSearch`,
1130
+ performs a POST request and uses its result to build a single :class:`~eodag.api.search_result.SearchResult`
1131
+ object.
1132
+
1133
+ The available configuration parameters are inherited from parent classes, with some a particularity
1134
+ for pagination for this plugin.
1135
+
1136
+ :param provider: An eodag providers configuration dictionary
1137
+ :param config: Search plugin configuration:
1138
+
1139
+ * :attr:`~eodag.config.PluginConfig.pagination` (:class:`~eodag.config.PluginConfig.Pagination`)
1140
+ (**mandatory**): The configuration of how the pagination is done on the provider. For
1141
+ this plugin it has the node:
1142
+
1143
+ * :attr:`~eodag.config.PluginConfig.Pagination.next_page_query_obj` (``str``): The
1144
+ additional parameters needed to perform search. These parameters won't be included in
1145
+ the result. This must be a json dict formatted like ``{{"foo":"bar"}}`` because it
1146
+ will be passed to a :meth:`str.format` method before being loaded as json.
1147
+ """
1148
+
1149
+ def collect_search_urls(
1150
+ self,
1151
+ prep: PreparedSearch = PreparedSearch(),
1152
+ **kwargs: Any,
1153
+ ) -> Tuple[List[str], int]:
1154
+ """Wraps PostJsonSearch.collect_search_urls to force product count to 1
1155
+
1156
+ :param prep: :class:`~eodag.plugins.search.PreparedSearch` object containing information for the search
1157
+ :param kwargs: keyword arguments used in the search
1158
+ :return: list of search url and number of results
1159
+ """
1160
+ urls, _ = super().collect_search_urls(prep, **kwargs)
1161
+ return urls, 1
1162
+
1163
+ def do_search(
1164
+ self, prep: PreparedSearch = PreparedSearch(items_per_page=None), **kwargs: Any
1165
+ ) -> List[Dict[str, Any]]:
1166
+ """Perform the actual search request, and return result in a single element.
1167
+
1168
+ :param prep: :class:`~eodag.plugins.search.PreparedSearch` object containing information for the search
1169
+ :param kwargs: keyword arguments to be used in the search
1170
+ :return: list containing the results from the provider in json format
1171
+ """
540
1172
 
541
- python_queryables = create_model("m", **field_definitions).model_fields
542
- return {**default_queryables, **model_fields_to_annotated(python_queryables)}
1173
+ prep.url = prep.search_urls[0]
1174
+ prep.info_message = f"Sending search request: {prep.url}"
1175
+ prep.exception_message = (
1176
+ f"Skipping error while searching for {self.provider}"
1177
+ f" {self.__class__.__name__} instance"
1178
+ )
1179
+ response = self._request(prep)
1180
+
1181
+ return [response.json()]
1182
+
1183
+ def build_query_string(
1184
+ self, product_type: str, **kwargs: Any
1185
+ ) -> Tuple[Dict[str, Any], str]:
1186
+ """Build The query string using the search parameters
1187
+
1188
+ :param product_type: product type id
1189
+ :param kwargs: keyword arguments to be used in the query string
1190
+ :return: formatted query params and encode query string
1191
+ """
1192
+ return QueryStringSearch.build_query_string(
1193
+ self, product_type=product_type, **kwargs
1194
+ )
1195
+
1196
+
1197
+ class WekeoECMWFSearch(ECMWFSearch):
1198
+ """
1199
+ WekeoECMWFSearch search plugin.
1200
+
1201
+ This plugin, which inherits from :class:`~eodag.plugins.search.build_search_result.ECMWFSearch`,
1202
+ performs a POST request and uses its result to build a single :class:`~eodag.api.search_result.SearchResult`
1203
+ object. In contrast to ECMWFSearch or MeteoblueSearch, the products are only build with information
1204
+ returned by the provider.
1205
+
1206
+ The available configuration parameters are inherited from parent classes, with some a particularity
1207
+ for pagination for this plugin.
1208
+
1209
+ :param provider: An eodag providers configuration dictionary
1210
+ :param config: Search plugin configuration:
1211
+
1212
+ * :attr:`~eodag.config.PluginConfig.pagination` (:class:`~eodag.config.PluginConfig.Pagination`)
1213
+ (**mandatory**): The configuration of how the pagination is done on the provider. For
1214
+ this plugin it has the node:
1215
+
1216
+ * :attr:`~eodag.config.PluginConfig.Pagination.next_page_query_obj` (``str``): The
1217
+ additional parameters needed to perform search. These parameters won't be included in
1218
+ the result. This must be a json dict formatted like ``{{"foo":"bar"}}`` because it
1219
+ will be passed to a :meth:`str.format` method before being loaded as json.
1220
+ """
1221
+
1222
+ def normalize_results(
1223
+ self, results: RawSearchResult, **kwargs: Any
1224
+ ) -> List[EOProduct]:
1225
+ """Build :class:`~eodag.api.product._product.EOProduct` from provider result
1226
+
1227
+ :param results: Raw provider result as single dict in list
1228
+ :param kwargs: Search arguments
1229
+ :returns: list of single :class:`~eodag.api.product._product.EOProduct`
1230
+ """
1231
+
1232
+ # formating of orderLink requires access to the productType value.
1233
+ results.data = [
1234
+ {**result, **results.product_type_def_params} for result in results
1235
+ ]
1236
+
1237
+ normalized = QueryStringSearch.normalize_results(self, results, **kwargs)
1238
+
1239
+ if not normalized:
1240
+ return normalized
1241
+
1242
+ query_params_encoded = quote_plus(orjson.dumps(results.query_params))
1243
+ for product in normalized:
1244
+ properties = {**product.properties, **results.query_params}
1245
+ properties["_dc_qs"] = query_params_encoded
1246
+ product.properties = {ecmwf_format(k): v for k, v in properties.items()}
1247
+
1248
+ return normalized
1249
+
1250
+ def do_search(self, *args: Any, **kwargs: Any) -> List[Dict[str, Any]]:
1251
+ """Should perform the actual search request.
1252
+
1253
+ :param args: arguments to be used in the search
1254
+ :param kwargs: keyword arguments to be used in the search
1255
+ :return: list containing the results from the provider in json format
1256
+ """
1257
+ return QueryStringSearch.do_search(self, *args, **kwargs)
1258
+
1259
+ def build_query_string(
1260
+ self, product_type: str, **kwargs: Any
1261
+ ) -> Tuple[Dict[str, Any], str]:
1262
+ """Build The query string using the search parameters
1263
+
1264
+ :param product_type: product type id
1265
+ :param kwargs: keyword arguments to be used in the query string
1266
+ :return: formatted query params and encode query string
1267
+ """
1268
+ # Reorder kwargs to make sure year/month/day/time if set overwrite default datetime.
1269
+ # strip_quotes to remove duplicated quotes like "'1_1'" produced by convertors like to_geojson.
1270
+ priority_keys = [
1271
+ "startTimeFromAscendingNode",
1272
+ "completionTimeFromAscendingNode",
1273
+ ]
1274
+ ordered_kwargs = {k: kwargs[k] for k in priority_keys if k in kwargs}
1275
+ ordered_kwargs.update({k: strip_quotes(v) for k, v in kwargs.items()})
1276
+
1277
+ return QueryStringSearch.build_query_string(
1278
+ self, product_type=product_type, **ordered_kwargs
1279
+ )