eodag 3.0.1__py3-none-any.whl → 3.1.0b2__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (87) hide show
  1. eodag/api/core.py +164 -127
  2. eodag/api/product/_assets.py +11 -11
  3. eodag/api/product/_product.py +45 -30
  4. eodag/api/product/drivers/__init__.py +81 -4
  5. eodag/api/product/drivers/base.py +65 -4
  6. eodag/api/product/drivers/generic.py +65 -0
  7. eodag/api/product/drivers/sentinel1.py +97 -0
  8. eodag/api/product/drivers/sentinel2.py +95 -0
  9. eodag/api/product/metadata_mapping.py +101 -85
  10. eodag/api/search_result.py +13 -23
  11. eodag/cli.py +26 -5
  12. eodag/config.py +78 -81
  13. eodag/plugins/apis/base.py +1 -1
  14. eodag/plugins/apis/ecmwf.py +46 -22
  15. eodag/plugins/apis/usgs.py +16 -15
  16. eodag/plugins/authentication/aws_auth.py +16 -13
  17. eodag/plugins/authentication/base.py +5 -3
  18. eodag/plugins/authentication/header.py +3 -3
  19. eodag/plugins/authentication/keycloak.py +4 -4
  20. eodag/plugins/authentication/oauth.py +7 -3
  21. eodag/plugins/authentication/openid_connect.py +16 -16
  22. eodag/plugins/authentication/sas_auth.py +4 -4
  23. eodag/plugins/authentication/token.py +41 -10
  24. eodag/plugins/authentication/token_exchange.py +1 -1
  25. eodag/plugins/base.py +4 -4
  26. eodag/plugins/crunch/base.py +4 -4
  27. eodag/plugins/crunch/filter_date.py +4 -4
  28. eodag/plugins/crunch/filter_latest_intersect.py +6 -6
  29. eodag/plugins/crunch/filter_latest_tpl_name.py +7 -7
  30. eodag/plugins/crunch/filter_overlap.py +4 -4
  31. eodag/plugins/crunch/filter_property.py +6 -7
  32. eodag/plugins/download/aws.py +58 -78
  33. eodag/plugins/download/base.py +38 -56
  34. eodag/plugins/download/creodias_s3.py +29 -0
  35. eodag/plugins/download/http.py +173 -183
  36. eodag/plugins/download/s3rest.py +10 -11
  37. eodag/plugins/manager.py +10 -20
  38. eodag/plugins/search/__init__.py +6 -5
  39. eodag/plugins/search/base.py +87 -44
  40. eodag/plugins/search/build_search_result.py +1067 -329
  41. eodag/plugins/search/cop_marine.py +22 -12
  42. eodag/plugins/search/creodias_s3.py +9 -73
  43. eodag/plugins/search/csw.py +11 -11
  44. eodag/plugins/search/data_request_search.py +16 -15
  45. eodag/plugins/search/qssearch.py +103 -187
  46. eodag/plugins/search/stac_list_assets.py +85 -0
  47. eodag/plugins/search/static_stac_search.py +3 -3
  48. eodag/resources/ext_product_types.json +1 -1
  49. eodag/resources/product_types.yml +663 -304
  50. eodag/resources/providers.yml +823 -1749
  51. eodag/resources/stac_api.yml +2 -2
  52. eodag/resources/user_conf_template.yml +11 -0
  53. eodag/rest/cache.py +2 -2
  54. eodag/rest/config.py +3 -3
  55. eodag/rest/core.py +112 -82
  56. eodag/rest/errors.py +5 -5
  57. eodag/rest/server.py +33 -14
  58. eodag/rest/stac.py +40 -38
  59. eodag/rest/types/collections_search.py +3 -3
  60. eodag/rest/types/eodag_search.py +29 -23
  61. eodag/rest/types/queryables.py +15 -16
  62. eodag/rest/types/stac_search.py +15 -25
  63. eodag/rest/utils/__init__.py +14 -21
  64. eodag/rest/utils/cql_evaluate.py +6 -6
  65. eodag/rest/utils/rfc3339.py +2 -2
  66. eodag/types/__init__.py +75 -28
  67. eodag/types/bbox.py +2 -2
  68. eodag/types/download_args.py +3 -3
  69. eodag/types/queryables.py +183 -72
  70. eodag/types/search_args.py +4 -4
  71. eodag/types/whoosh.py +127 -3
  72. eodag/utils/__init__.py +152 -50
  73. eodag/utils/exceptions.py +28 -21
  74. eodag/utils/import_system.py +2 -2
  75. eodag/utils/repr.py +65 -6
  76. eodag/utils/requests.py +13 -13
  77. eodag/utils/rest.py +2 -2
  78. eodag/utils/s3.py +208 -0
  79. eodag/utils/stac_reader.py +10 -10
  80. {eodag-3.0.1.dist-info → eodag-3.1.0b2.dist-info}/METADATA +77 -76
  81. eodag-3.1.0b2.dist-info/RECORD +113 -0
  82. {eodag-3.0.1.dist-info → eodag-3.1.0b2.dist-info}/WHEEL +1 -1
  83. {eodag-3.0.1.dist-info → eodag-3.1.0b2.dist-info}/entry_points.txt +4 -2
  84. eodag/utils/constraints.py +0 -244
  85. eodag-3.0.1.dist-info/RECORD +0 -109
  86. {eodag-3.0.1.dist-info → eodag-3.1.0b2.dist-info}/LICENSE +0 -0
  87. {eodag-3.0.1.dist-info → eodag-3.1.0b2.dist-info}/top_level.txt +0 -0
@@ -17,21 +17,13 @@
17
17
  # limitations under the License.
18
18
  from __future__ import annotations
19
19
 
20
+ import functools
20
21
  import hashlib
21
22
  import logging
22
- from datetime import datetime, timedelta, timezone
23
- from typing import (
24
- TYPE_CHECKING,
25
- Annotated,
26
- Any,
27
- Dict,
28
- List,
29
- Optional,
30
- Set,
31
- Tuple,
32
- cast,
33
- get_args,
34
- )
23
+ import re
24
+ from collections import OrderedDict
25
+ from datetime import datetime, timedelta
26
+ from typing import TYPE_CHECKING, Annotated, Any, Optional, Union, cast
35
27
  from urllib.parse import quote_plus, unquote_plus
36
28
 
37
29
  import geojson
@@ -39,235 +31,329 @@ import orjson
39
31
  from dateutil.parser import isoparse
40
32
  from dateutil.tz import tzutc
41
33
  from jsonpath_ng import Child, Fields, Root
42
- from pydantic import create_model
34
+ from pydantic import Field
43
35
  from pydantic.fields import FieldInfo
36
+ from requests.auth import AuthBase
37
+ from shapely.geometry.base import BaseGeometry
38
+ from typing_extensions import get_args
44
39
 
45
40
  from eodag.api.product import EOProduct
46
41
  from eodag.api.product.metadata_mapping import (
47
42
  NOT_AVAILABLE,
48
43
  NOT_MAPPED,
49
- get_queryable_from_provider,
44
+ format_metadata,
45
+ format_query_params,
50
46
  mtd_cfg_as_conversion_and_querypath,
51
47
  properties_from_json,
52
48
  )
53
49
  from eodag.api.search_result import RawSearchResult
54
50
  from eodag.plugins.search import PreparedSearch
55
- from eodag.plugins.search.base import Search
56
- from eodag.plugins.search.qssearch import PostJsonSearch
57
- from eodag.types import json_field_definition_to_python, model_fields_to_annotated
58
- from eodag.types.queryables import CommonQueryables
51
+ from eodag.plugins.search.qssearch import PostJsonSearch, QueryStringSearch
52
+ from eodag.types import json_field_definition_to_python
53
+ from eodag.types.queryables import Queryables, QueryablesDict
59
54
  from eodag.utils import (
60
- DEFAULT_MISSION_START_DATE,
55
+ DEFAULT_SEARCH_TIMEOUT,
61
56
  deepcopy,
62
57
  dict_items_recursive_sort,
63
58
  get_geometry_from_various,
64
- )
65
- from eodag.utils.constraints import (
66
- fetch_constraints,
67
- get_constraint_queryables_with_additional_params,
59
+ is_range_in_range,
68
60
  )
69
61
  from eodag.utils.exceptions import ValidationError
62
+ from eodag.utils.requests import fetch_json
70
63
 
71
64
  if TYPE_CHECKING:
72
65
  from eodag.config import PluginConfig
73
66
 
74
67
  logger = logging.getLogger("eodag.search.build_search_result")
75
68
 
76
-
77
- class BuildPostSearchResult(PostJsonSearch):
78
- """BuildPostSearchResult search plugin.
79
-
80
- This plugin, which inherits from :class:`~eodag.plugins.search.qssearch.PostJsonSearch`,
81
- performs a POST request and uses its result to build a single :class:`~eodag.api.search_result.SearchResult`
82
- object.
83
-
84
- The available configuration parameters are inherited from parent classes
85
- (:class:`~eodag.plugins.search.qssearch.PostJsonSearch` and
86
- :class:`~eodag.plugins.search.qssearch.QueryStringSearch`), with particularly for this plugin:
87
-
88
- :param provider: provider name
89
- :param config: Search plugin configuration:
90
-
91
- * :attr:`~eodag.config.PluginConfig.remove_from_query` (``List[str]``): List of parameters
92
- used to parse metadata but that must not be included to the query
93
-
69
+ # keywords from ECMWF keyword database + "dataset" (not part of database but exists)
70
+ # database: https://confluence.ecmwf.int/display/UDOC/Keywords+in+MARS+and+Dissemination+requests
71
+ ECMWF_KEYWORDS = [
72
+ "dataset",
73
+ "accuracy",
74
+ "activity",
75
+ "anoffset",
76
+ "bitmap",
77
+ "block",
78
+ "channel",
79
+ "class",
80
+ "database",
81
+ "date",
82
+ "diagnostic",
83
+ "direction",
84
+ "domain",
85
+ "duplicates",
86
+ "expect",
87
+ "expver",
88
+ "fcmonth",
89
+ "fcperiod",
90
+ "fieldset",
91
+ "filter",
92
+ "format",
93
+ "frame",
94
+ "frequency",
95
+ "generation",
96
+ "grid",
97
+ "hdate",
98
+ "ident",
99
+ "interpolation",
100
+ "intgrid",
101
+ "iteration",
102
+ "latitude",
103
+ "levelist",
104
+ "levtype",
105
+ "longitude",
106
+ "lsm",
107
+ "method",
108
+ "number",
109
+ "obsgroup",
110
+ "obstype",
111
+ "origin",
112
+ "packing",
113
+ "padding",
114
+ "param",
115
+ "priority",
116
+ "product",
117
+ "range",
118
+ "realization",
119
+ "refdate",
120
+ "reference",
121
+ "reportype",
122
+ "repres",
123
+ "resolution",
124
+ "rotation",
125
+ "section",
126
+ "source",
127
+ "step",
128
+ "stream",
129
+ "system",
130
+ "target",
131
+ "time",
132
+ "truncation",
133
+ "type",
134
+ "use",
135
+ ]
136
+
137
+ # additional keywords from copernicus services
138
+ COP_DS_KEYWORDS = [
139
+ "aerosol_type",
140
+ "altitude",
141
+ "product_type",
142
+ "band",
143
+ "cdr_type",
144
+ "data_format",
145
+ "dataset_type",
146
+ "day",
147
+ "download_format",
148
+ "ensemble_member",
149
+ "experiment",
150
+ "forcing_type",
151
+ "gcm",
152
+ "hday",
153
+ "hmonth",
154
+ "horizontal_resolution",
155
+ "hydrological_model",
156
+ "hydrological_year",
157
+ "hyear",
158
+ "input_observations",
159
+ "leadtime_hour",
160
+ "leadtime_month",
161
+ "level",
162
+ "location",
163
+ "model",
164
+ "model_level",
165
+ "model_levels",
166
+ "month",
167
+ "nominal_day",
168
+ "originating_centre",
169
+ "period",
170
+ "pressure_level",
171
+ "processing_level",
172
+ "processing_type",
173
+ "product_version",
174
+ "quantity",
175
+ "rcm",
176
+ "region",
177
+ "release_version",
178
+ "satellite",
179
+ "sensor",
180
+ "sensor_and_algorithm",
181
+ "soil_level",
182
+ "sky_type",
183
+ "statistic",
184
+ "system_version",
185
+ "temporal_aggregation",
186
+ "time_aggregation",
187
+ "time_reference",
188
+ "time_step",
189
+ "variable",
190
+ "variable_type",
191
+ "version",
192
+ "year",
193
+ ]
194
+
195
+
196
+ def keywords_to_mdt(
197
+ keywords: list[str], prefix: Optional[str] = None
198
+ ) -> dict[str, Any]:
94
199
  """
200
+ Make metadata mapping dict from a list of keywords
95
201
 
96
- def count_hits(
97
- self, count_url: Optional[str] = None, result_type: Optional[str] = None
98
- ) -> int:
99
- """Count method that will always return 1."""
100
- return 1
101
-
102
- def collect_search_urls(
103
- self,
104
- prep: PreparedSearch = PreparedSearch(),
105
- **kwargs: Any,
106
- ) -> Tuple[List[str], int]:
107
- """Wraps PostJsonSearch.collect_search_urls to force product count to 1"""
108
- urls, _ = super(BuildPostSearchResult, self).collect_search_urls(prep, **kwargs)
109
- return urls, 1
110
-
111
- def do_search(
112
- self, prep: PreparedSearch = PreparedSearch(items_per_page=None), **kwargs: Any
113
- ) -> List[Dict[str, Any]]:
114
- """Perform the actual search request, and return result in a single element."""
115
- prep.url = prep.search_urls[0]
116
- prep.info_message = f"Sending search request: {prep.url}"
117
- prep.exception_message = (
118
- f"Skipping error while searching for {self.provider}"
119
- f" {self.__class__.__name__} instance"
120
- )
121
- response = self._request(prep)
122
-
123
- return [response.json()]
124
-
125
- def normalize_results(
126
- self, results: RawSearchResult, **kwargs: Any
127
- ) -> List[EOProduct]:
128
- """Build :class:`~eodag.api.product._product.EOProduct` from provider result
129
-
130
- :param results: Raw provider result as single dict in list
131
- :param kwargs: Search arguments
132
- :returns: list of single :class:`~eodag.api.product._product.EOProduct`
133
- """
134
- product_type = kwargs.get("productType")
202
+ prefix:keyword:
203
+ - keyword
204
+ - $."prefix:keyword"
135
205
 
136
- result = results[0]
206
+ >>> keywords_to_mdt(["month", "year"])
207
+ {'month': ['month', '$."month"'], 'year': ['year', '$."year"']}
208
+ >>> keywords_to_mdt(["month", "year"], "ecmwf")
209
+ {'ecmwf:month': ['month', '$."ecmwf:month"'], 'ecmwf:year': ['year', '$."ecmwf:year"']}
137
210
 
138
- # datacube query string got from previous search
139
- _dc_qs = kwargs.pop("_dc_qs", None)
140
- if _dc_qs is not None:
141
- qs = unquote_plus(unquote_plus(_dc_qs))
142
- sorted_unpaginated_query_params = geojson.loads(qs)
143
- else:
144
- # update result with query parameters without pagination (or search-only params)
145
- if isinstance(
146
- self.config.pagination["next_page_query_obj"], str
147
- ) and hasattr(results, "query_params_unpaginated"):
148
- unpaginated_query_params = results.query_params_unpaginated
149
- elif isinstance(self.config.pagination["next_page_query_obj"], str):
150
- next_page_query_obj = orjson.loads(
151
- self.config.pagination["next_page_query_obj"].format()
152
- )
153
- unpaginated_query_params = {
154
- k: v[0] if (isinstance(v, list) and len(v) == 1) else v
155
- for k, v in results.query_params.items()
156
- if (k, v) not in next_page_query_obj.items()
157
- }
158
- else:
159
- unpaginated_query_params = self.query_params
160
-
161
- # query hash, will be used to build a product id
162
- sorted_unpaginated_query_params = dict_items_recursive_sort(
163
- unpaginated_query_params
164
- )
165
-
166
- # use all available query_params to parse properties
167
- result = dict(
168
- result,
169
- **sorted_unpaginated_query_params,
170
- qs=sorted_unpaginated_query_params,
171
- )
172
-
173
- # remove unwanted query params
174
- for param in getattr(self.config, "remove_from_query", []):
175
- sorted_unpaginated_query_params.pop(param, None)
176
-
177
- qs = geojson.dumps(sorted_unpaginated_query_params)
178
-
179
- query_hash = hashlib.sha1(str(qs).encode("UTF-8")).hexdigest()
180
-
181
- # update result with product_type_def_params and search args if not None (and not auth)
182
- kwargs.pop("auth", None)
183
- result.update(results.product_type_def_params)
184
- result = dict(result, **{k: v for k, v in kwargs.items() if v is not None})
185
-
186
- # parse properties
187
- parsed_properties = properties_from_json(
188
- result,
189
- self.config.metadata_mapping,
190
- discovery_config=getattr(self.config, "discover_metadata", {}),
191
- )
192
-
193
- if not product_type:
194
- product_type = parsed_properties.get("productType", None)
195
-
196
- # build product id
197
- id_prefix = (product_type or self.provider).upper()
198
- product_id = "%s_%s_%s_%s" % (
199
- id_prefix,
200
- parsed_properties["startTimeFromAscendingNode"]
201
- .split("T")[0]
202
- .replace("-", ""),
203
- parsed_properties["completionTimeFromAscendingNode"]
204
- .split("T")[0]
205
- .replace("-", ""),
206
- query_hash,
211
+ :param keywords: List of keywords to be converted
212
+ :param prefix: prefix to be added to the parameter in the mapping
213
+ :return: metadata mapping dict
214
+ """
215
+ mdt: dict[str, Any] = {}
216
+ for keyword in keywords:
217
+ key = f"{prefix}:{keyword}" if prefix else keyword
218
+ mdt[key] = [keyword, f'$."{key}"']
219
+ return mdt
220
+
221
+
222
+ def strip_quotes(value: Any) -> Any:
223
+ """Strip superfluous quotes from elements (added by mapping converter to_geojson).
224
+
225
+ >>> strip_quotes("'abc'")
226
+ 'abc'
227
+ >>> strip_quotes(["'abc'", '"def'])
228
+ ['abc', 'def']
229
+ >>> strip_quotes({"'abc'": 'def"'})
230
+ {'abc': 'def'}
231
+
232
+ :param value: value from which quotes should be removed (should be either str or list)
233
+ :return: value without quotes
234
+ :raises: NotImplementedError
235
+ """
236
+ if isinstance(value, (list, tuple)):
237
+ return [strip_quotes(v) for v in value]
238
+ elif isinstance(value, dict):
239
+ return {strip_quotes(k): strip_quotes(v) for k, v in value.items()}
240
+ else:
241
+ return str(value).strip("'\"")
242
+
243
+
244
+ def _update_properties_from_element(
245
+ prop: dict[str, Any], element: dict[str, Any], values: list[str]
246
+ ) -> None:
247
+ """updates a property dict with the given values based on the information from the element dict
248
+ e.g. the type is set based on the type of the element
249
+ """
250
+ # multichoice elements are transformed into array
251
+ if element["type"] in ("StringListWidget", "StringListArrayWidget"):
252
+ prop["type"] = "array"
253
+ if values:
254
+ prop["items"] = {"type": "string", "enum": sorted(values)}
255
+
256
+ # single choice elements are transformed into string
257
+ elif element["type"] in (
258
+ "StringChoiceWidget",
259
+ "DateRangeWidget",
260
+ "FreeformInputWidget",
261
+ ):
262
+ prop["type"] = "string"
263
+ if values:
264
+ prop["enum"] = sorted(values)
265
+
266
+ # a bbox element
267
+ elif element["type"] in ["GeographicExtentWidget", "GeographicExtentMapWidget"]:
268
+ prop.update(
269
+ {
270
+ "type": "array",
271
+ "minItems": 4,
272
+ "additionalItems": False,
273
+ "items": [
274
+ {
275
+ "type": "number",
276
+ "maximum": 180,
277
+ "minimum": -180,
278
+ "description": "West border of the bounding box",
279
+ },
280
+ {
281
+ "type": "number",
282
+ "maximum": 90,
283
+ "minimum": -90,
284
+ "description": "South border of the bounding box",
285
+ },
286
+ {
287
+ "type": "number",
288
+ "maximum": 180,
289
+ "minimum": -180,
290
+ "description": "East border of the bounding box",
291
+ },
292
+ {
293
+ "type": "number",
294
+ "maximum": 90,
295
+ "minimum": -90,
296
+ "description": "North border of the bounding box",
297
+ },
298
+ ],
299
+ }
207
300
  )
208
- parsed_properties["id"] = parsed_properties["title"] = product_id
209
-
210
- # update downloadLink and orderLink
211
- parsed_properties["_dc_qs"] = quote_plus(qs)
212
- if parsed_properties["downloadLink"] != "Not Available":
213
- parsed_properties["downloadLink"] += f"?{qs}"
214
301
 
215
- # parse metadata needing downloadLink
216
- dl_path = Fields("downloadLink")
217
- dl_path_from_root = Child(Root(), dl_path)
218
- for param, mapping in self.config.metadata_mapping.items():
219
- if dl_path in mapping or dl_path_from_root in mapping:
220
- parsed_properties.update(
221
- properties_from_json(parsed_properties, {param: mapping})
222
- )
302
+ # DateRangeWidget is a calendar date picker
303
+ if element["type"] == "DateRangeWidget":
304
+ prop["description"] = "date formatted like yyyy-mm-dd/yyyy-mm-dd"
223
305
 
224
- # use product_type_config as default properties
225
- parsed_properties = dict(
226
- getattr(self.config, "product_type_config", {}),
227
- **parsed_properties,
228
- )
306
+ if description := element.get("help"):
307
+ prop["description"] = description
229
308
 
230
- product = EOProduct(
231
- provider=self.provider,
232
- productType=product_type,
233
- properties=parsed_properties,
234
- )
235
309
 
236
- return [
237
- product,
238
- ]
310
+ def ecmwf_format(v: str) -> str:
311
+ """Add ECMWF prefix to value v if v is a ECMWF keyword."""
312
+ return "ecmwf:" + v if v in ECMWF_KEYWORDS + COP_DS_KEYWORDS else v
239
313
 
240
314
 
241
- class BuildSearchResult(BuildPostSearchResult):
242
- """BuildSearchResult search plugin.
315
+ class ECMWFSearch(PostJsonSearch):
316
+ """ECMWF search plugin.
243
317
 
244
- This plugin builds a single :class:`~eodag.api.search_result.SearchResult` object
318
+ This plugin builds a :class:`~eodag.api.search_result.SearchResult` containing a single product
245
319
  using given query parameters as product properties.
246
320
 
247
- The available configuration parameters inherits from parent classes
248
- (:class:`~eodag.plugins.search.build_search_result.BuildPostSearchResult`,
249
- :class:`~eodag.plugins.search.qssearch.PostJsonSearch` and
250
- :class:`~eodag.plugins.search.qssearch.QueryStringSearch`), with particularly for this plugin:
321
+ The available configuration parameters inherits from parent classes, with some particular parameters
322
+ for this plugin.
251
323
 
252
- :param provider: provider name
324
+ :param provider: An eodag providers configuration dictionary
253
325
  :param config: Search plugin configuration:
254
326
 
255
- * :attr:`~eodag.config.PluginConfig.end_date_excluded` (``bool``): Set to ``False`` if provider
256
- does not include end date in the search request; In this case, if the end date is at midnight,
257
- the previous day will be used. default: ``True``
258
-
327
+ * :attr:`~eodag.config.PluginConfig.remove_from_query` (``list[str]``): List of parameters
328
+ used to parse metadata but that must not be included to the query
329
+ * :attr:`~eodag.config.PluginConfig.end_date_excluded` (``bool``): Set to `False` if
330
+ provider does not include end date to search
331
+ * :attr:`~eodag.config.PluginConfig.discover_queryables`
332
+ (:class:`~eodag.config.PluginConfig.DiscoverQueryables`): configuration to fetch the queryables from a
333
+ provider queryables endpoint; It has the following keys:
334
+
335
+ * :attr:`~eodag.config.PluginConfig.DiscoverQueryables.fetch_url` (``str``): url to fetch the queryables valid
336
+ for all product types
337
+ * :attr:`~eodag.config.PluginConfig.DiscoverQueryables.product_type_fetch_url` (``str``): url to fetch the
338
+ queryables for a specific product type
339
+ * :attr:`~eodag.config.PluginConfig.DiscoverQueryables.constraints_url` (``str``): url of the constraint file
340
+ used to build queryables
259
341
  """
260
342
 
261
343
  def __init__(self, provider: str, config: PluginConfig) -> None:
262
- # init self.config.metadata_mapping using Search Base plugin
263
- Search.__init__(self, provider, config)
344
+ config.metadata_mapping = {
345
+ **keywords_to_mdt(ECMWF_KEYWORDS + COP_DS_KEYWORDS, "ecmwf"),
346
+ **config.metadata_mapping,
347
+ }
348
+
349
+ super().__init__(provider, config)
264
350
 
265
351
  self.config.__dict__.setdefault("api_endpoint", "")
266
352
 
267
353
  # needed by QueryStringSearch.build_query_string / format_free_text_search
268
354
  self.config.__dict__.setdefault("free_text_search_operations", {})
269
355
  # needed for compatibility
270
- self.config.__dict__.setdefault("pagination", {"next_page_query_obj": "{{}}"})
356
+ self.config.pagination.setdefault("next_page_query_obj", "{{}}")
271
357
 
272
358
  # parse jsonpath on init: product type specific metadata-mapping
273
359
  for product_type in self.config.products.keys():
@@ -304,46 +390,70 @@ class BuildSearchResult(BuildPostSearchResult):
304
390
  "metadata_mapping"
305
391
  ] = product_type_metadata_mapping
306
392
 
307
- def do_search(self, *args: Any, **kwargs: Any) -> List[Dict[str, Any]]:
308
- """Should perform the actual search request."""
393
+ def do_search(self, *args: Any, **kwargs: Any) -> list[dict[str, Any]]:
394
+ """Should perform the actual search request.
395
+
396
+ :param args: arguments to be used in the search
397
+ :param kwargs: keyword arguments to be used in the search
398
+ :return: list containing the results from the provider in json format
399
+ """
400
+ # no real search. We fake it all
309
401
  return [{}]
310
402
 
311
403
  def query(
312
404
  self,
313
405
  prep: PreparedSearch = PreparedSearch(),
314
406
  **kwargs: Any,
315
- ) -> Tuple[List[EOProduct], Optional[int]]:
316
- """Build ready-to-download SearchResult"""
407
+ ) -> tuple[list[EOProduct], Optional[int]]:
408
+ """Build ready-to-download SearchResult
317
409
 
318
- self._preprocess_search_params(kwargs)
410
+ :param prep: :class:`~eodag.plugins.search.PreparedSearch` object containing information needed for the search
411
+ :param kwargs: keyword arguments to be used in the search
412
+ :returns: list of products and number of products (optional)
413
+ """
414
+ product_type = prep.product_type
415
+ if not product_type:
416
+ product_type = kwargs.get("productType", None)
417
+ self._preprocess_search_params(kwargs, product_type)
418
+ result, num_items = super().query(prep, **kwargs)
419
+ if prep.count and not num_items:
420
+ num_items = 1
319
421
 
320
- return BuildPostSearchResult.query(self, prep, **kwargs)
422
+ return result, num_items
321
423
 
322
424
  def clear(self) -> None:
323
425
  """Clear search context"""
324
- pass
426
+ super().clear()
325
427
 
326
428
  def build_query_string(
327
429
  self, product_type: str, **kwargs: Any
328
- ) -> Tuple[Dict[str, Any], str]:
329
- """Build The query string using the search parameters"""
430
+ ) -> tuple[dict[str, Any], str]:
431
+ """Build The query string using the search parameters
432
+
433
+ :param product_type: product type id
434
+ :param kwargs: keyword arguments to be used in the query string
435
+ :return: formatted query params and encode query string
436
+ """
330
437
  # parse kwargs as properties as they might be needed to build the query
331
438
  parsed_properties = properties_from_json(
332
439
  kwargs,
333
440
  self.config.metadata_mapping,
334
441
  )
335
442
  available_properties = {
336
- k: v
443
+ # We strip values of superfluous quotes (added by mapping converter to_geojson).
444
+ k: strip_quotes(v)
337
445
  for k, v in parsed_properties.items()
338
446
  if v not in [NOT_AVAILABLE, NOT_MAPPED]
339
447
  }
340
448
 
341
449
  # build and return the query
342
- return BuildPostSearchResult.build_query_string(
343
- self, product_type=product_type, **available_properties
450
+ return super().build_query_string(
451
+ product_type=product_type, **available_properties
344
452
  )
345
453
 
346
- def _preprocess_search_params(self, params: Dict[str, Any]) -> None:
454
+ def _preprocess_search_params(
455
+ self, params: dict[str, Any], product_type: Optional[str]
456
+ ) -> None:
347
457
  """Preprocess search parameters before making a request to the CDS API.
348
458
 
349
459
  This method is responsible for checking and updating the provided search parameters
@@ -352,6 +462,7 @@ class BuildSearchResult(BuildPostSearchResult):
352
462
  in the input parameters, default values or values from the configuration are used.
353
463
 
354
464
  :param params: Search parameters to be preprocessed.
465
+ :param product_type: (optional) product type id
355
466
  """
356
467
  _dc_qs = params.get("_dc_qs", None)
357
468
  if _dc_qs is not None:
@@ -378,165 +489,792 @@ class BuildSearchResult(BuildPostSearchResult):
378
489
  non_none_params = {k: v for k, v in params.items() if v}
379
490
 
380
491
  # productType
381
- dataset = params.get("dataset", None)
492
+ dataset = params.get("ecmwf:dataset", None)
382
493
  params["productType"] = non_none_params.get("productType", dataset)
383
494
 
384
495
  # dates
385
- mission_start_dt = datetime.fromisoformat(
386
- self.get_product_type_cfg_value(
387
- "missionStartDate", DEFAULT_MISSION_START_DATE
388
- ).replace(
389
- "Z", "+00:00"
390
- ) # before 3.11
391
- )
392
-
393
- default_end_from_cfg = self.config.products.get(params["productType"], {}).get(
394
- "_default_end_date", None
395
- )
396
- default_end_str = (
397
- default_end_from_cfg
398
- or (
399
- datetime.now(timezone.utc)
400
- if params.get("startTimeFromAscendingNode")
401
- else mission_start_dt + timedelta(days=1)
402
- ).isoformat()
403
- )
404
-
405
- params["startTimeFromAscendingNode"] = non_none_params.get(
406
- "startTimeFromAscendingNode", mission_start_dt.isoformat()
407
- )
408
- params["completionTimeFromAscendingNode"] = non_none_params.get(
409
- "completionTimeFromAscendingNode", default_end_str
410
- )
496
+ # check if default dates have to be added
497
+ if getattr(self.config, "dates_required", False):
498
+ self._check_date_params(params, product_type)
411
499
 
412
500
  # adapt end date if it is midnight
413
- end_date_excluded = getattr(self.config, "end_date_excluded", True)
414
- is_datetime = True
415
- try:
416
- end_date = datetime.strptime(
417
- params["completionTimeFromAscendingNode"], "%Y-%m-%dT%H:%M:%SZ"
418
- )
419
- end_date = end_date.replace(tzinfo=tzutc())
420
- except ValueError:
501
+ if "completionTimeFromAscendingNode" in params:
502
+ end_date_excluded = getattr(self.config, "end_date_excluded", True)
503
+ is_datetime = True
421
504
  try:
422
505
  end_date = datetime.strptime(
423
- params["completionTimeFromAscendingNode"], "%Y-%m-%dT%H:%M:%S.%fZ"
506
+ params["completionTimeFromAscendingNode"], "%Y-%m-%dT%H:%M:%SZ"
424
507
  )
425
508
  end_date = end_date.replace(tzinfo=tzutc())
426
509
  except ValueError:
427
- end_date = isoparse(params["completionTimeFromAscendingNode"])
428
- is_datetime = False
429
- start_date = isoparse(params["startTimeFromAscendingNode"])
430
- if (
431
- not end_date_excluded
432
- and is_datetime
433
- and end_date > start_date
434
- and end_date == end_date.replace(hour=0, minute=0, second=0, microsecond=0)
435
- ):
436
- end_date += timedelta(days=-1)
437
- params["completionTimeFromAscendingNode"] = end_date.isoformat()
510
+ try:
511
+ end_date = datetime.strptime(
512
+ params["completionTimeFromAscendingNode"],
513
+ "%Y-%m-%dT%H:%M:%S.%fZ",
514
+ )
515
+ end_date = end_date.replace(tzinfo=tzutc())
516
+ except ValueError:
517
+ end_date = isoparse(params["completionTimeFromAscendingNode"])
518
+ is_datetime = False
519
+ start_date = isoparse(params["startTimeFromAscendingNode"])
520
+ if (
521
+ not end_date_excluded
522
+ and is_datetime
523
+ and end_date > start_date
524
+ and end_date
525
+ == end_date.replace(hour=0, minute=0, second=0, microsecond=0)
526
+ ):
527
+ end_date += timedelta(days=-1)
528
+ params["completionTimeFromAscendingNode"] = end_date.isoformat()
438
529
 
439
530
  # geometry
440
531
  if "geometry" in params:
441
532
  params["geometry"] = get_geometry_from_various(geometry=params["geometry"])
442
533
 
534
+ def _get_product_type_queryables(
535
+ self, product_type: Optional[str], alias: Optional[str], filters: dict[str, Any]
536
+ ) -> QueryablesDict:
537
+ """Override to set additional_properties to false."""
538
+ default_values: dict[str, Any] = deepcopy(
539
+ getattr(self.config, "products", {}).get(product_type, {})
540
+ )
541
+ default_values.pop("metadata_mapping", None)
542
+
543
+ filters["productType"] = product_type
544
+ queryables = self.discover_queryables(**{**default_values, **filters}) or {}
545
+
546
+ return QueryablesDict(additional_properties=False, **queryables)
547
+
443
548
  def discover_queryables(
444
549
  self, **kwargs: Any
445
- ) -> Optional[Dict[str, Annotated[Any, FieldInfo]]]:
550
+ ) -> Optional[dict[str, Annotated[Any, FieldInfo]]]:
446
551
  """Fetch queryables list from provider using its constraints file
447
552
 
448
553
  :param kwargs: additional filters for queryables (`productType` and other search
449
554
  arguments)
450
555
  :returns: fetched queryable parameters dict
451
556
  """
452
- constraints_file_url = getattr(self.config, "constraints_file_url", "")
453
- if not constraints_file_url:
454
- return {}
455
- product_type = kwargs.pop("productType", None)
456
- if not product_type:
457
- return {}
557
+ product_type = kwargs.pop("productType")
558
+ product_type_config = self.config.products.get(product_type, {})
559
+ provider_product_type = (
560
+ product_type_config.get("ecmwf:dataset", None)
561
+ or product_type_config["productType"]
562
+ )
563
+ if "start" in kwargs:
564
+ kwargs["startTimeFromAscendingNode"] = kwargs.pop("start")
565
+ if "end" in kwargs:
566
+ kwargs["completionTimeFromAscendingNode"] = kwargs.pop("end")
567
+
568
+ # extract default datetime
569
+ processed_kwargs = deepcopy(kwargs)
570
+ self._preprocess_search_params(processed_kwargs, product_type)
571
+
572
+ constraints_url = format_metadata(
573
+ getattr(self.config, "discover_queryables", {}).get("constraints_url", ""),
574
+ **kwargs,
575
+ )
576
+ constraints: list[dict[str, Any]] = self._fetch_data(constraints_url)
458
577
 
459
- provider_product_type = self.config.products.get(product_type, {}).get(
460
- "dataset", None
578
+ form_url = format_metadata(
579
+ getattr(self.config, "discover_queryables", {}).get("form_url", ""),
580
+ **kwargs,
461
581
  )
462
- user_provider_product_type = kwargs.pop("dataset", None)
582
+ form: list[dict[str, Any]] = self._fetch_data(form_url)
583
+
584
+ formated_kwargs = self.format_as_provider_keyword(
585
+ product_type, processed_kwargs
586
+ )
587
+ # we re-apply kwargs input to consider override of year, month, day and time.
588
+ for key in kwargs:
589
+ if key.startswith("ecmwf:"):
590
+ formated_kwargs[key.replace("ecmwf:", "")] = kwargs[key]
591
+ elif key in (
592
+ "startTimeFromAscendingNode",
593
+ "completionTimeFromAscendingNode",
594
+ "geom",
595
+ ):
596
+ formated_kwargs[key] = kwargs[key]
597
+ else:
598
+ raise ValidationError(
599
+ f"{key} is not a queryable parameter for {self.provider}"
600
+ )
601
+
602
+ # we use non empty kwargs as default to integrate user inputs
603
+ # it is needed because pydantic json schema does not represent "value"
604
+ # but only "default"
605
+ non_empty_formated: dict[str, Any] = {
606
+ k: v
607
+ for k, v in formated_kwargs.items()
608
+ if v and (not isinstance(v, list) or all(v))
609
+ }
610
+ non_empty_kwargs: dict[str, Any] = {
611
+ k: v
612
+ for k, v in processed_kwargs.items()
613
+ if v and (not isinstance(v, list) or all(v))
614
+ }
615
+
616
+ required_keywords: set[str] = set()
617
+
618
+ # calculate available values
619
+ if constraints:
620
+ # Apply constraints filtering
621
+ available_values = self.available_values_from_constraints(
622
+ constraints,
623
+ non_empty_formated,
624
+ form_keywords=[f["name"] for f in form],
625
+ )
626
+
627
+ # Pre-compute the required keywords (present in all constraint dicts)
628
+ # when form, required keywords are extracted directly from form
629
+ if not form:
630
+ required_keywords = set(constraints[0].keys())
631
+ for constraint in constraints[1:]:
632
+ required_keywords.intersection_update(constraint.keys())
633
+ else:
634
+ values_url = getattr(self.config, "available_values_url", "")
635
+ if not values_url:
636
+ return self.queryables_from_metadata_mapping(product_type)
637
+ if "{" in values_url:
638
+ values_url = values_url.format(productType=provider_product_type)
639
+ data = self._fetch_data(values_url)
640
+ available_values = data["constraints"]
641
+ required_keywords = data.get("required", [])
642
+
643
+ # To check if all keywords are queryable parameters, we check if they are in the
644
+ # available values or the product type config (available values calculated from the
645
+ # constraints might not include all queryables)
646
+ for keyword in kwargs:
647
+ if (
648
+ keyword
649
+ not in available_values.keys()
650
+ | product_type_config.keys()
651
+ | {
652
+ "startTimeFromAscendingNode",
653
+ "completionTimeFromAscendingNode",
654
+ "geom",
655
+ }
656
+ and keyword not in [f["name"] for f in form]
657
+ and keyword.replace("ecmwf:", "")
658
+ not in set(list(available_values.keys()) + [f["name"] for f in form])
659
+ ):
660
+ raise ValidationError(f"{keyword} is not a queryable parameter")
661
+
662
+ # generate queryables
663
+ if form:
664
+ queryables = self.queryables_by_form(
665
+ form,
666
+ available_values,
667
+ non_empty_formated,
668
+ )
669
+ else:
670
+ queryables = self.queryables_by_values(
671
+ available_values, list(required_keywords), non_empty_kwargs
672
+ )
673
+
674
+ # ecmwf:date is replaced by start and end.
675
+ # start and end filters are supported whenever combinations of "year", "month", "day" filters exist
463
676
  if (
464
- user_provider_product_type
465
- and user_provider_product_type != provider_product_type
677
+ queryables.pop("ecmwf:date", None)
678
+ or "ecmwf:year" in queryables
679
+ or "ecmwf:hyear" in queryables
466
680
  ):
467
- raise ValidationError(
468
- f"Cannot change dataset from {provider_product_type} to {user_provider_product_type}"
681
+ queryables.update(
682
+ {
683
+ "start": Queryables.get_with_default(
684
+ "start", non_empty_kwargs.get("startTimeFromAscendingNode")
685
+ ),
686
+ "end": Queryables.get_with_default(
687
+ "end",
688
+ non_empty_kwargs.get("completionTimeFromAscendingNode"),
689
+ ),
690
+ }
469
691
  )
470
692
 
471
- # defaults
472
- default_queryables = self._get_defaults_as_queryables(product_type)
473
- # remove dataset from queryables
474
- default_queryables.pop("dataset", None)
693
+ # area is geom in EODAG.
694
+ if queryables.pop("area", None):
695
+ queryables["geom"] = Annotated[
696
+ Union[str, dict[str, float], BaseGeometry],
697
+ Field(
698
+ None,
699
+ description="Read EODAG documentation for all supported geometry format.",
700
+ ),
701
+ ]
475
702
 
476
- non_empty_kwargs = {k: v for k, v in kwargs.items() if v}
703
+ return queryables
477
704
 
478
- if "{" in constraints_file_url:
479
- constraints_file_url = constraints_file_url.format(
480
- dataset=provider_product_type
481
- )
482
- constraints = fetch_constraints(constraints_file_url, self)
483
- if not constraints:
484
- return default_queryables
485
-
486
- constraint_params: Dict[str, Dict[str, Set[Any]]] = {}
487
- if len(kwargs) == 0:
488
- # get values from constraints without additional filters
489
- for constraint in constraints:
490
- for key in constraint.keys():
491
- if key in constraint_params:
492
- constraint_params[key]["enum"].update(constraint[key])
493
- else:
494
- constraint_params[key] = {}
495
- constraint_params[key]["enum"] = set(constraint[key])
496
- else:
497
- # get values from constraints with additional filters
498
- constraints_input_params = {k: v for k, v in non_empty_kwargs.items()}
499
- constraint_params = get_constraint_queryables_with_additional_params(
500
- constraints, constraints_input_params, self, product_type
705
+ def available_values_from_constraints(
706
+ self,
707
+ constraints: list[dict[str, Any]],
708
+ input_keywords: dict[str, Any],
709
+ form_keywords: list[str],
710
+ ) -> dict[str, list[str]]:
711
+ """
712
+ Filter constraints using input_keywords. Return list of available queryables.
713
+ All constraint entries must have the same parameters.
714
+
715
+ :param constraints: list of constraints received from the provider
716
+ :param input_keywords: dict of input parameters given by the user
717
+ :param form_keywords: list of keyword names from the provider form endpoint
718
+ :return: dict with available values for each parameter
719
+ """
720
+ # get ordered constraint keywords
721
+ constraints_keywords = list(
722
+ OrderedDict.fromkeys(k for c in constraints for k in c.keys())
723
+ )
724
+
725
+ # prepare ordered input keywords formatted as provider's keywords
726
+ # required to filter with constraints
727
+ ordered_keywords = (
728
+ [kw for kw in form_keywords if kw in constraints_keywords]
729
+ if form_keywords
730
+ else constraints_keywords
731
+ )
732
+
733
+ # filter constraint entries matching input keyword values
734
+ filtered_constraints: list[dict[str, Any]]
735
+
736
+ parsed_keywords: list[str] = []
737
+ for keyword in ordered_keywords:
738
+ values = input_keywords.get(keyword)
739
+
740
+ if values is None:
741
+ parsed_keywords.append(keyword)
742
+ continue
743
+
744
+ # we only compare list of strings.
745
+ if isinstance(values, dict):
746
+ raise ValidationError(
747
+ f"Parameter value as object is not supported: {keyword}={values}"
748
+ )
749
+ filter_v = values if isinstance(values, (list, tuple)) else [values]
750
+
751
+ # We convert every single value to a list of string
752
+ # We strip values of superfluous quotes (added by mapping converter to_geojson).
753
+ # ECMWF accept values with /to/. We need to split it to an array
754
+ # ECMWF accept values in format val1/val2. We need to split it to an array
755
+ sep = re.compile(r"/to/|/")
756
+ filter_v = [i for v in filter_v for i in sep.split(strip_quotes(v))]
757
+
758
+ # special handling for time 0000 converted to 0 by pre-formating with metadata_mapping
759
+ if keyword.split(":")[-1] == "time":
760
+ filter_v = ["0000" if str(v) == "0" else v for v in filter_v]
761
+
762
+ # Collect missing values to report errors
763
+ missing_values = set(filter_v)
764
+
765
+ # Filter constraints and check for missing values
766
+ filtered_constraints = []
767
+ for entry in constraints:
768
+ # Filter based on the presence of any value in filter_v
769
+ entry_values = entry.get(keyword, [])
770
+
771
+ # date constraint may be intervals. We identify intervals with a "/" in the value
772
+ # we assume that if the first value is an interval, all values are intervals
773
+ present_values = []
774
+ if keyword == "date" and "/" in entry[keyword][0]:
775
+ if any(is_range_in_range(x, values[0]) for x in entry[keyword]):
776
+ present_values = filter_v
777
+ else:
778
+ present_values = [
779
+ value for value in filter_v if value in entry_values
780
+ ]
781
+
782
+ # Remove present values from the missing_values set
783
+ missing_values -= set(present_values)
784
+
785
+ if present_values:
786
+ filtered_constraints.append(entry)
787
+
788
+ # raise an error as no constraint entry matched the input keywords
789
+ # raise an error if one value from input is not allowed
790
+ if not filtered_constraints or missing_values:
791
+ allowed_values = list(
792
+ {value for c in constraints for value in c.get(keyword, [])}
793
+ )
794
+ # restore ecmwf: prefix before raising error
795
+ keyword = f"ecmwf:{keyword}"
796
+
797
+ all_keywords_str = ""
798
+ if len(parsed_keywords) > 1:
799
+ keywords = [
800
+ f"ecmwf:{k}={pk}"
801
+ for k in parsed_keywords
802
+ if (pk := input_keywords.get(k))
803
+ ]
804
+ all_keywords_str = f" with {', '.join(keywords)}"
805
+
806
+ raise ValidationError(
807
+ f"{keyword}={values} is not available"
808
+ f"{all_keywords_str}."
809
+ f" Allowed values are {', '.join(allowed_values)}."
810
+ )
811
+
812
+ parsed_keywords.append(keyword)
813
+ constraints = filtered_constraints
814
+
815
+ available_values: dict[str, Any] = {k: set() for k in ordered_keywords}
816
+
817
+ # we aggregate the constraint entries left
818
+ for entry in constraints:
819
+ for key, value in entry.items():
820
+ available_values[key].update(value)
821
+
822
+ return {k: list(v) for k, v in available_values.items()}
823
+
824
+ def queryables_by_form(
825
+ self,
826
+ form: list[dict[str, Any]],
827
+ available_values: dict[str, list[str]],
828
+ defaults: dict[str, Any],
829
+ ) -> dict[str, Annotated[Any, FieldInfo]]:
830
+ """
831
+ Generate Annotated field definitions from form entries and available values
832
+ Used by Copernicus services like cop_cds, cop_ads, cop_ewds.
833
+
834
+ :param form: data fetched from the form endpoint of the provider
835
+ :param available_values: available values for each parameter
836
+ :param defaults: default values for the parameters
837
+ :return: dict of annotated queryables
838
+ """
839
+ queryables: dict[str, Annotated[Any, FieldInfo]] = {}
840
+
841
+ required_list: list[str] = []
842
+ for element in form:
843
+ name: str = element["name"]
844
+
845
+ # those are not parameter elements.
846
+ if name in ("area_group", "global", "warning", "licences"):
847
+ continue
848
+ if "type" not in element or element["type"] == "FreeEditionWidget":
849
+ continue
850
+
851
+ # ordering done by id -> set id to high value if not present -> element will be last
852
+ if "id" not in element:
853
+ element["id"] = 100
854
+
855
+ prop = {"title": element.get("label", name)}
856
+
857
+ details = element.get("details", {})
858
+
859
+ # add values from form if keyword was not in constraints
860
+ values = (
861
+ available_values[name]
862
+ if name in available_values
863
+ else details.get("values")
501
864
  )
502
- # query params that are not in constraints but might be default queryables
503
- if len(constraint_params) == 1 and "not_available" in constraint_params:
504
- not_queryables: Set[str] = set()
505
- for constraint_param in constraint_params["not_available"]["enum"]:
506
- param = CommonQueryables.get_queryable_from_alias(constraint_param)
507
- if param in dict(
508
- CommonQueryables.model_fields, **default_queryables
509
- ):
510
- non_empty_kwargs.pop(constraint_param)
511
- else:
512
- not_queryables.add(constraint_param)
513
- if not_queryables:
514
- raise ValidationError(
515
- f"parameter(s) {not_queryables} not queryable"
865
+
866
+ # updates the properties with the values given based on the information from the element
867
+ _update_properties_from_element(prop, element, values)
868
+
869
+ default = defaults.get(name)
870
+
871
+ if details:
872
+ fields = details.get("fields")
873
+ if fields and (comment := fields[0].get("comment")):
874
+ prop["description"] = comment
875
+
876
+ if name == "area" and isinstance(default, dict):
877
+ default = list(default.values())
878
+
879
+ if default:
880
+ # We strip values of superfluous quotes (addded by mapping converter to_geojson).
881
+ default = strip_quotes(default)
882
+
883
+ # sometimes form returns default as array instead of string
884
+ if default and prop["type"] == "string" and isinstance(default, list):
885
+ default = ",".join(default)
886
+
887
+ is_required = bool(element.get("required"))
888
+ if is_required:
889
+ required_list.append(name)
890
+
891
+ queryables[ecmwf_format(name)] = Annotated[
892
+ get_args(
893
+ json_field_definition_to_python(
894
+ prop,
895
+ default_value=default,
896
+ required=is_required,
516
897
  )
517
- else:
518
- # get constraints again without common queryables
519
- constraint_params = (
520
- get_constraint_queryables_with_additional_params(
521
- constraints, non_empty_kwargs, self, product_type
522
- )
898
+ )
899
+ ]
900
+
901
+ return queryables
902
+
903
+ def queryables_by_values(
904
+ self,
905
+ available_values: dict[str, list[str]],
906
+ required_keywords: list[str],
907
+ defaults: dict[str, Any],
908
+ ) -> dict[str, Annotated[Any, FieldInfo]]:
909
+ """
910
+ Generate Annotated field definitions from available values.
911
+ Used by ECMWF data providers like dedt_lumi.
912
+
913
+ :param available_values: available values for each parameter
914
+ :param required_keywords: list of required parameters
915
+ :param defaults: default values for the parameters
916
+ :return: dict of annotated queryables
917
+ """
918
+ # Rename keywords from form with metadata mapping.
919
+ # Needed to map constraints like "xxxx" to eodag parameter "ecmwf:xxxx"
920
+ required = [ecmwf_format(k) for k in required_keywords]
921
+
922
+ queryables: dict[str, Annotated[Any, FieldInfo]] = {}
923
+ for name, values in available_values.items():
924
+ # Rename keywords from form with metadata mapping.
925
+ # Needed to map constraints like "xxxx" to eodag parameter "ecmwf:xxxx"
926
+ key = ecmwf_format(name)
927
+
928
+ default = defaults.get(key)
929
+
930
+ queryables[key] = Annotated[
931
+ get_args(
932
+ json_field_definition_to_python(
933
+ {"type": "string", "title": name, "enum": values},
934
+ default_value=strip_quotes(default) if default else None,
935
+ required=bool(key in required),
523
936
  )
937
+ )
938
+ ]
939
+
940
+ return queryables
941
+
942
+ def format_as_provider_keyword(
943
+ self, product_type: str, properties: dict[str, Any]
944
+ ) -> dict[str, Any]:
945
+ """Return provider equivalent keyword names from EODAG keywords.
946
+
947
+ :param product_type: product type id
948
+ :param properties: dict of properties to be formatted
949
+ :return: dict of formatted properties
950
+ """
951
+ parsed_properties = properties_from_json(
952
+ properties,
953
+ self.config.metadata_mapping,
954
+ )
955
+ available_properties = {
956
+ k: v
957
+ for k, v in parsed_properties.items()
958
+ if v not in [NOT_AVAILABLE, NOT_MAPPED]
959
+ }
960
+ return format_query_params(product_type, self.config, available_properties)
961
+
962
+ def _fetch_data(self, url: str) -> Any:
963
+ """
964
+ fetches from a provider elements like constraints or forms.
965
+
966
+ :param url: url from which the constraints can be fetched
967
+ :returns: json file content fetched from the provider
968
+ """
969
+ if not url:
970
+ return []
971
+
972
+ auth = (
973
+ self.auth
974
+ if hasattr(self, "auth") and isinstance(self.auth, AuthBase)
975
+ else None
976
+ )
977
+ timeout = getattr(self.config, "timeout", DEFAULT_SEARCH_TIMEOUT)
978
+ return functools.lru_cache()(fetch_json)(url, auth=auth, timeout=timeout)
979
+
980
+ def normalize_results(
981
+ self, results: RawSearchResult, **kwargs: Any
982
+ ) -> list[EOProduct]:
983
+ """Build :class:`~eodag.api.product._product.EOProduct` from provider result
984
+
985
+ :param results: Raw provider result as single dict in list
986
+ :param kwargs: Search arguments
987
+ :returns: list of single :class:`~eodag.api.product._product.EOProduct`
988
+ """
524
989
 
525
- field_definitions: Dict[str, Any] = {}
526
- for json_param, json_mtd in constraint_params.items():
527
- param = (
528
- get_queryable_from_provider(
529
- json_param, self.get_metadata_mapping(product_type)
990
+ product_type = kwargs.get("productType")
991
+
992
+ result = results[0]
993
+
994
+ # datacube query string got from previous search
995
+ _dc_qs = kwargs.pop("_dc_qs", None)
996
+ if _dc_qs is not None:
997
+ qs = unquote_plus(unquote_plus(_dc_qs))
998
+ sorted_unpaginated_query_params = geojson.loads(qs)
999
+ else:
1000
+ # update result with query parameters without pagination (or search-only params)
1001
+ if isinstance(
1002
+ self.config.pagination["next_page_query_obj"], str
1003
+ ) and hasattr(results, "query_params_unpaginated"):
1004
+ unpaginated_query_params = results.query_params_unpaginated
1005
+ elif isinstance(self.config.pagination["next_page_query_obj"], str):
1006
+ next_page_query_obj = orjson.loads(
1007
+ self.config.pagination["next_page_query_obj"].format()
530
1008
  )
531
- or json_param
1009
+ unpaginated_query_params = {
1010
+ k: v
1011
+ for k, v in results.query_params.items()
1012
+ if (k, v) not in next_page_query_obj.items()
1013
+ }
1014
+ else:
1015
+ unpaginated_query_params = self.query_params
1016
+ # query hash, will be used to build a product id
1017
+ sorted_unpaginated_query_params = dict_items_recursive_sort(
1018
+ unpaginated_query_params
532
1019
  )
533
- default = kwargs.get(param, None) or self.config.products.get(
534
- product_type, {}
535
- ).get(param, None)
536
- annotated_def = json_field_definition_to_python(
537
- json_mtd, default_value=default, required=True
1020
+
1021
+ # use all available query_params to parse properties
1022
+ result = dict(
1023
+ result,
1024
+ **sorted_unpaginated_query_params,
1025
+ qs=sorted_unpaginated_query_params,
1026
+ )
1027
+
1028
+ # remove unwanted query params
1029
+ for param in getattr(self.config, "remove_from_query", []):
1030
+ sorted_unpaginated_query_params.pop(param, None)
1031
+
1032
+ qs = geojson.dumps(sorted_unpaginated_query_params)
1033
+
1034
+ query_hash = hashlib.sha1(str(qs).encode("UTF-8")).hexdigest()
1035
+
1036
+ # update result with product_type_def_params and search args if not None (and not auth)
1037
+ kwargs.pop("auth", None)
1038
+ result.update(results.product_type_def_params)
1039
+ result = dict(result, **{k: v for k, v in kwargs.items() if v is not None})
1040
+
1041
+ # parse properties
1042
+ parsed_properties = properties_from_json(
1043
+ result,
1044
+ self.config.metadata_mapping,
1045
+ discovery_config=getattr(self.config, "discover_metadata", {}),
1046
+ )
1047
+
1048
+ if not product_type:
1049
+ product_type = parsed_properties.get("productType", None)
1050
+
1051
+ # build product id
1052
+ id_prefix = (product_type or self.provider).upper()
1053
+ if (
1054
+ "startTimeFromAscendingNode" in parsed_properties
1055
+ and parsed_properties["startTimeFromAscendingNode"] != "Not Available"
1056
+ and "completionTimeFromAscendingNode" in parsed_properties
1057
+ and parsed_properties["completionTimeFromAscendingNode"] != "Not Available"
1058
+ ):
1059
+ product_id = "%s_%s_%s_%s" % (
1060
+ id_prefix,
1061
+ parsed_properties["startTimeFromAscendingNode"]
1062
+ .split("T")[0]
1063
+ .replace("-", ""),
1064
+ parsed_properties["completionTimeFromAscendingNode"]
1065
+ .split("T")[0]
1066
+ .replace("-", ""),
1067
+ query_hash,
538
1068
  )
539
- field_definitions[param] = get_args(annotated_def)
1069
+ elif (
1070
+ "startTimeFromAscendingNode" in parsed_properties
1071
+ and parsed_properties["startTimeFromAscendingNode"] != "Not Available"
1072
+ ):
1073
+ product_id = "%s_%s_%s" % (
1074
+ id_prefix,
1075
+ parsed_properties["startTimeFromAscendingNode"]
1076
+ .split("T")[0]
1077
+ .replace("-", ""),
1078
+ query_hash,
1079
+ )
1080
+ else:
1081
+ product_id = f"{id_prefix}_{query_hash}"
1082
+
1083
+ parsed_properties["id"] = parsed_properties["title"] = product_id
1084
+
1085
+ # update downloadLink and orderLink
1086
+ parsed_properties["_dc_qs"] = quote_plus(qs)
1087
+ if parsed_properties["downloadLink"] != "Not Available":
1088
+ parsed_properties["downloadLink"] += f"?{qs}"
1089
+
1090
+ # parse metadata needing downloadLink
1091
+ dl_path = Fields("downloadLink")
1092
+ dl_path_from_root = Child(Root(), dl_path)
1093
+ for param, mapping in self.config.metadata_mapping.items():
1094
+ if dl_path in mapping or dl_path_from_root in mapping:
1095
+ parsed_properties.update(
1096
+ properties_from_json(parsed_properties, {param: mapping})
1097
+ )
1098
+
1099
+ # use product_type_config as default properties
1100
+ parsed_properties = dict(
1101
+ getattr(self.config, "product_type_config", {}),
1102
+ **parsed_properties,
1103
+ )
1104
+
1105
+ product = EOProduct(
1106
+ provider=self.provider,
1107
+ productType=product_type,
1108
+ properties=parsed_properties,
1109
+ )
1110
+
1111
+ return [
1112
+ product,
1113
+ ]
540
1114
 
541
- python_queryables = create_model("m", **field_definitions).model_fields
542
- return {**default_queryables, **model_fields_to_annotated(python_queryables)}
1115
+ def count_hits(
1116
+ self, count_url: Optional[str] = None, result_type: Optional[str] = None
1117
+ ) -> int:
1118
+ """Count method that will always return 1.
1119
+
1120
+ :param count_url: not used, only here because this method overwrites count_hits from the parent class
1121
+ :param result_type: not used, only here because this method overwrites count_hits from the parent class
1122
+ :return: always 1
1123
+ """
1124
+ return 1
1125
+
1126
+
1127
+ class MeteoblueSearch(ECMWFSearch):
1128
+ """MeteoblueSearch search plugin.
1129
+
1130
+ This plugin, which inherits from :class:`~eodag.plugins.search.build_search_result.ECMWFSearch`,
1131
+ performs a POST request and uses its result to build a single :class:`~eodag.api.search_result.SearchResult`
1132
+ object.
1133
+
1134
+ The available configuration parameters are inherited from parent classes, with some a particularity
1135
+ for pagination for this plugin.
1136
+
1137
+ :param provider: An eodag providers configuration dictionary
1138
+ :param config: Search plugin configuration:
1139
+
1140
+ * :attr:`~eodag.config.PluginConfig.pagination` (:class:`~eodag.config.PluginConfig.Pagination`)
1141
+ (**mandatory**): The configuration of how the pagination is done on the provider. For
1142
+ this plugin it has the node:
1143
+
1144
+ * :attr:`~eodag.config.PluginConfig.Pagination.next_page_query_obj` (``str``): The
1145
+ additional parameters needed to perform search. These parameters won't be included in
1146
+ the result. This must be a json dict formatted like ``{{"foo":"bar"}}`` because it
1147
+ will be passed to a :meth:`str.format` method before being loaded as json.
1148
+ """
1149
+
1150
+ def collect_search_urls(
1151
+ self,
1152
+ prep: PreparedSearch = PreparedSearch(),
1153
+ **kwargs: Any,
1154
+ ) -> tuple[list[str], int]:
1155
+ """Wraps PostJsonSearch.collect_search_urls to force product count to 1
1156
+
1157
+ :param prep: :class:`~eodag.plugins.search.PreparedSearch` object containing information for the search
1158
+ :param kwargs: keyword arguments used in the search
1159
+ :return: list of search url and number of results
1160
+ """
1161
+ urls, _ = super().collect_search_urls(prep, **kwargs)
1162
+ return urls, 1
1163
+
1164
+ def do_search(
1165
+ self, prep: PreparedSearch = PreparedSearch(items_per_page=None), **kwargs: Any
1166
+ ) -> list[dict[str, Any]]:
1167
+ """Perform the actual search request, and return result in a single element.
1168
+
1169
+ :param prep: :class:`~eodag.plugins.search.PreparedSearch` object containing information for the search
1170
+ :param kwargs: keyword arguments to be used in the search
1171
+ :return: list containing the results from the provider in json format
1172
+ """
1173
+
1174
+ prep.url = prep.search_urls[0]
1175
+ prep.info_message = f"Sending search request: {prep.url}"
1176
+ prep.exception_message = (
1177
+ f"Skipping error while searching for {self.provider}"
1178
+ f" {self.__class__.__name__} instance"
1179
+ )
1180
+ response = self._request(prep)
1181
+
1182
+ return [response.json()]
1183
+
1184
+ def build_query_string(
1185
+ self, product_type: str, **kwargs: Any
1186
+ ) -> tuple[dict[str, Any], str]:
1187
+ """Build The query string using the search parameters
1188
+
1189
+ :param product_type: product type id
1190
+ :param kwargs: keyword arguments to be used in the query string
1191
+ :return: formatted query params and encode query string
1192
+ """
1193
+ return QueryStringSearch.build_query_string(
1194
+ self, product_type=product_type, **kwargs
1195
+ )
1196
+
1197
+
1198
+ class WekeoECMWFSearch(ECMWFSearch):
1199
+ """
1200
+ WekeoECMWFSearch search plugin.
1201
+
1202
+ This plugin, which inherits from :class:`~eodag.plugins.search.build_search_result.ECMWFSearch`,
1203
+ performs a POST request and uses its result to build a single :class:`~eodag.api.search_result.SearchResult`
1204
+ object. In contrast to ECMWFSearch or MeteoblueSearch, the products are only build with information
1205
+ returned by the provider.
1206
+
1207
+ The available configuration parameters are inherited from parent classes, with some a particularity
1208
+ for pagination for this plugin.
1209
+
1210
+ :param provider: An eodag providers configuration dictionary
1211
+ :param config: Search plugin configuration:
1212
+
1213
+ * :attr:`~eodag.config.PluginConfig.pagination` (:class:`~eodag.config.PluginConfig.Pagination`)
1214
+ (**mandatory**): The configuration of how the pagination is done on the provider. For
1215
+ this plugin it has the node:
1216
+
1217
+ * :attr:`~eodag.config.PluginConfig.Pagination.next_page_query_obj` (``str``): The
1218
+ additional parameters needed to perform search. These parameters won't be included in
1219
+ the result. This must be a json dict formatted like ``{{"foo":"bar"}}`` because it
1220
+ will be passed to a :meth:`str.format` method before being loaded as json.
1221
+ """
1222
+
1223
+ def normalize_results(
1224
+ self, results: RawSearchResult, **kwargs: Any
1225
+ ) -> list[EOProduct]:
1226
+ """Build :class:`~eodag.api.product._product.EOProduct` from provider result
1227
+
1228
+ :param results: Raw provider result as single dict in list
1229
+ :param kwargs: Search arguments
1230
+ :returns: list of single :class:`~eodag.api.product._product.EOProduct`
1231
+ """
1232
+
1233
+ # formating of orderLink requires access to the productType value.
1234
+ results.data = [
1235
+ {**result, **results.product_type_def_params} for result in results
1236
+ ]
1237
+
1238
+ normalized = QueryStringSearch.normalize_results(self, results, **kwargs)
1239
+
1240
+ if not normalized:
1241
+ return normalized
1242
+
1243
+ query_params_encoded = quote_plus(orjson.dumps(results.query_params))
1244
+ for product in normalized:
1245
+ properties = {**product.properties, **results.query_params}
1246
+ properties["_dc_qs"] = query_params_encoded
1247
+ product.properties = {ecmwf_format(k): v for k, v in properties.items()}
1248
+
1249
+ return normalized
1250
+
1251
+ def do_search(self, *args: Any, **kwargs: Any) -> list[dict[str, Any]]:
1252
+ """Should perform the actual search request.
1253
+
1254
+ :param args: arguments to be used in the search
1255
+ :param kwargs: keyword arguments to be used in the search
1256
+ :return: list containing the results from the provider in json format
1257
+ """
1258
+ return QueryStringSearch.do_search(self, *args, **kwargs)
1259
+
1260
+ def build_query_string(
1261
+ self, product_type: str, **kwargs: Any
1262
+ ) -> tuple[dict[str, Any], str]:
1263
+ """Build The query string using the search parameters
1264
+
1265
+ :param product_type: product type id
1266
+ :param kwargs: keyword arguments to be used in the query string
1267
+ :return: formatted query params and encode query string
1268
+ """
1269
+ # Reorder kwargs to make sure year/month/day/time if set overwrite default datetime.
1270
+ # strip_quotes to remove duplicated quotes like "'1_1'" produced by convertors like to_geojson.
1271
+ priority_keys = [
1272
+ "startTimeFromAscendingNode",
1273
+ "completionTimeFromAscendingNode",
1274
+ ]
1275
+ ordered_kwargs = {k: kwargs[k] for k in priority_keys if k in kwargs}
1276
+ ordered_kwargs.update({k: strip_quotes(v) for k, v in kwargs.items()})
1277
+
1278
+ return QueryStringSearch.build_query_string(
1279
+ self, product_type=product_type, **ordered_kwargs
1280
+ )