eodag 4.0.0a1__py3-none-any.whl → 4.0.0a2__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- eodag/__init__.py +6 -1
- eodag/api/collection.py +353 -0
- eodag/api/core.py +308 -296
- eodag/api/product/_product.py +15 -29
- eodag/api/product/drivers/__init__.py +2 -42
- eodag/api/product/drivers/base.py +0 -11
- eodag/api/product/metadata_mapping.py +34 -5
- eodag/api/search_result.py +144 -9
- eodag/cli.py +18 -15
- eodag/config.py +37 -3
- eodag/plugins/apis/ecmwf.py +16 -4
- eodag/plugins/apis/usgs.py +18 -7
- eodag/plugins/crunch/filter_latest_intersect.py +1 -0
- eodag/plugins/crunch/filter_overlap.py +3 -7
- eodag/plugins/search/__init__.py +3 -0
- eodag/plugins/search/base.py +6 -6
- eodag/plugins/search/build_search_result.py +157 -56
- eodag/plugins/search/cop_marine.py +48 -8
- eodag/plugins/search/csw.py +18 -8
- eodag/plugins/search/qssearch.py +331 -88
- eodag/plugins/search/static_stac_search.py +11 -12
- eodag/resources/collections.yml +610 -348
- eodag/resources/ext_collections.json +1 -1
- eodag/resources/ext_product_types.json +1 -1
- eodag/resources/providers.yml +330 -58
- eodag/resources/stac_provider.yml +4 -2
- eodag/resources/user_conf_template.yml +9 -0
- eodag/types/__init__.py +2 -0
- eodag/types/queryables.py +16 -0
- eodag/utils/__init__.py +47 -2
- eodag/utils/repr.py +2 -0
- {eodag-4.0.0a1.dist-info → eodag-4.0.0a2.dist-info}/METADATA +4 -2
- {eodag-4.0.0a1.dist-info → eodag-4.0.0a2.dist-info}/RECORD +37 -36
- {eodag-4.0.0a1.dist-info → eodag-4.0.0a2.dist-info}/WHEEL +0 -0
- {eodag-4.0.0a1.dist-info → eodag-4.0.0a2.dist-info}/entry_points.txt +0 -0
- {eodag-4.0.0a1.dist-info → eodag-4.0.0a2.dist-info}/licenses/LICENSE +0 -0
- {eodag-4.0.0a1.dist-info → eodag-4.0.0a2.dist-info}/top_level.txt +0 -0
eodag/plugins/apis/ecmwf.py
CHANGED
|
@@ -27,6 +27,7 @@ from ecmwfapi import ECMWFDataServer, ECMWFService
|
|
|
27
27
|
from ecmwfapi.api import APIException, Connection, get_apikey_values
|
|
28
28
|
from pydantic.fields import FieldInfo
|
|
29
29
|
|
|
30
|
+
from eodag.api.search_result import RawSearchResult
|
|
30
31
|
from eodag.plugins.apis.base import Api
|
|
31
32
|
from eodag.plugins.search import PreparedSearch
|
|
32
33
|
from eodag.plugins.search.base import Search
|
|
@@ -103,15 +104,25 @@ class EcmwfApi(Api, ECMWFSearch):
|
|
|
103
104
|
self.config.__dict__.setdefault("pagination", {"next_page_query_obj": "{{}}"})
|
|
104
105
|
self.config.__dict__.setdefault("api_endpoint", "")
|
|
105
106
|
|
|
106
|
-
def do_search(
|
|
107
|
+
def do_search(
|
|
108
|
+
self, prep: PreparedSearch = PreparedSearch(items_per_page=None), **kwargs: Any
|
|
109
|
+
) -> RawSearchResult:
|
|
107
110
|
"""Should perform the actual search request."""
|
|
108
|
-
|
|
111
|
+
raw_search_results = RawSearchResult([{}])
|
|
112
|
+
raw_search_results.search_params = kwargs
|
|
113
|
+
raw_search_results.query_params = (
|
|
114
|
+
prep.query_params if hasattr(prep, "query_params") else {}
|
|
115
|
+
)
|
|
116
|
+
raw_search_results.collection_def_params = (
|
|
117
|
+
prep.collection_def_params if hasattr(prep, "collection_def_params") else {}
|
|
118
|
+
)
|
|
119
|
+
return raw_search_results
|
|
109
120
|
|
|
110
121
|
def query(
|
|
111
122
|
self,
|
|
112
123
|
prep: PreparedSearch = PreparedSearch(),
|
|
113
124
|
**kwargs: Any,
|
|
114
|
-
) ->
|
|
125
|
+
) -> SearchResult:
|
|
115
126
|
"""Build ready-to-download SearchResult"""
|
|
116
127
|
|
|
117
128
|
# check collection, dates, geometry, use defaults if not specified
|
|
@@ -286,7 +297,8 @@ class EcmwfApi(Api, ECMWFSearch):
|
|
|
286
297
|
pass
|
|
287
298
|
|
|
288
299
|
def discover_queryables(
|
|
289
|
-
self,
|
|
300
|
+
self,
|
|
301
|
+
**kwargs: Any,
|
|
290
302
|
) -> Optional[dict[str, Annotated[Any, FieldInfo]]]:
|
|
291
303
|
"""Fetch queryables list from provider using metadata mapping
|
|
292
304
|
|
eodag/plugins/apis/usgs.py
CHANGED
|
@@ -34,6 +34,7 @@ from eodag.api.product.metadata_mapping import (
|
|
|
34
34
|
mtd_cfg_as_conversion_and_querypath,
|
|
35
35
|
properties_from_json,
|
|
36
36
|
)
|
|
37
|
+
from eodag.api.search_result import SearchResult
|
|
37
38
|
from eodag.plugins.apis.base import Api
|
|
38
39
|
from eodag.plugins.search import PreparedSearch
|
|
39
40
|
from eodag.utils import (
|
|
@@ -59,7 +60,6 @@ if TYPE_CHECKING:
|
|
|
59
60
|
from mypy_boto3_s3 import S3ServiceResource
|
|
60
61
|
from requests.auth import AuthBase
|
|
61
62
|
|
|
62
|
-
from eodag.api.search_result import SearchResult
|
|
63
63
|
from eodag.config import PluginConfig
|
|
64
64
|
from eodag.types.download_args import DownloadConf
|
|
65
65
|
from eodag.utils import DownloadedCallback, Unpack
|
|
@@ -138,14 +138,19 @@ class UsgsApi(Api):
|
|
|
138
138
|
self,
|
|
139
139
|
prep: PreparedSearch = PreparedSearch(),
|
|
140
140
|
**kwargs: Any,
|
|
141
|
-
) ->
|
|
141
|
+
) -> SearchResult:
|
|
142
142
|
"""Search for data on USGS catalogues"""
|
|
143
|
-
|
|
143
|
+
token = (
|
|
144
|
+
int(prep.next_page_token)
|
|
145
|
+
if prep.next_page_token is not None
|
|
146
|
+
else DEFAULT_PAGE
|
|
147
|
+
)
|
|
144
148
|
items_per_page = (
|
|
145
149
|
prep.items_per_page
|
|
146
|
-
|
|
147
|
-
|
|
150
|
+
or kwargs.pop("max_results", None)
|
|
151
|
+
or DEFAULT_ITEMS_PER_PAGE
|
|
148
152
|
)
|
|
153
|
+
search_params = {"items_per_page": items_per_page} | kwargs
|
|
149
154
|
collection = kwargs.get("collection")
|
|
150
155
|
if collection is None:
|
|
151
156
|
raise NoMatchingCollection(
|
|
@@ -196,7 +201,7 @@ class UsgsApi(Api):
|
|
|
196
201
|
ll=lower_left,
|
|
197
202
|
ur=upper_right,
|
|
198
203
|
max_results=items_per_page,
|
|
199
|
-
starting_number=
|
|
204
|
+
starting_number=token,
|
|
200
205
|
)
|
|
201
206
|
|
|
202
207
|
# search by id
|
|
@@ -294,7 +299,13 @@ class UsgsApi(Api):
|
|
|
294
299
|
else:
|
|
295
300
|
total_results = 0
|
|
296
301
|
|
|
297
|
-
|
|
302
|
+
formated_result = SearchResult(
|
|
303
|
+
final,
|
|
304
|
+
total_results,
|
|
305
|
+
search_params=search_params,
|
|
306
|
+
next_page_token=results["data"]["nextRecord"],
|
|
307
|
+
)
|
|
308
|
+
return formated_result
|
|
298
309
|
|
|
299
310
|
def download(
|
|
300
311
|
self,
|
|
@@ -70,6 +70,7 @@ class FilterLatestIntersect(Crunch):
|
|
|
70
70
|
# Warning: May crash if start_datetime is not in the appropriate format
|
|
71
71
|
products.sort(key=self.sort_product_by_start_date, reverse=True)
|
|
72
72
|
filtered: list[EOProduct] = []
|
|
73
|
+
search_extent: BaseGeometry
|
|
73
74
|
add_to_filtered = filtered.append
|
|
74
75
|
footprint: Union[dict[str, Any], BaseGeometry, Any] = search_params.get(
|
|
75
76
|
"geometry"
|
|
@@ -20,15 +20,11 @@ from __future__ import annotations
|
|
|
20
20
|
import logging
|
|
21
21
|
from typing import TYPE_CHECKING, Any
|
|
22
22
|
|
|
23
|
+
from shapely.errors import ShapelyError
|
|
24
|
+
|
|
23
25
|
from eodag.plugins.crunch.base import Crunch
|
|
24
26
|
from eodag.utils import get_geometry_from_various
|
|
25
27
|
|
|
26
|
-
try:
|
|
27
|
-
from shapely.errors import GEOSException
|
|
28
|
-
except ImportError:
|
|
29
|
-
# shapely < 2.0 compatibility
|
|
30
|
-
from shapely.errors import TopologicalError as GEOSException
|
|
31
|
-
|
|
32
28
|
if TYPE_CHECKING:
|
|
33
29
|
from eodag.api.product import EOProduct
|
|
34
30
|
|
|
@@ -108,7 +104,7 @@ class FilterOverlap(Crunch):
|
|
|
108
104
|
product_geometry = product.geometry.buffer(0)
|
|
109
105
|
try:
|
|
110
106
|
intersection = search_geom.intersection(product_geometry)
|
|
111
|
-
except
|
|
107
|
+
except ShapelyError:
|
|
112
108
|
logger.debug(
|
|
113
109
|
"Product geometry still invalid. Overlap test restricted to containment"
|
|
114
110
|
)
|
eodag/plugins/search/__init__.py
CHANGED
|
@@ -45,6 +45,8 @@ class PreparedSearch:
|
|
|
45
45
|
url: Optional[str] = None
|
|
46
46
|
info_message: Optional[str] = None
|
|
47
47
|
exception_message: Optional[str] = None
|
|
48
|
+
next_page_token: Optional[str] = None
|
|
49
|
+
next_page_token_key: Optional[str] = None
|
|
48
50
|
|
|
49
51
|
need_count: bool = field(init=False, repr=False)
|
|
50
52
|
query_params: dict[str, Any] = field(init=False, repr=False)
|
|
@@ -53,3 +55,4 @@ class PreparedSearch:
|
|
|
53
55
|
collection_def_params: dict[str, Any] = field(init=False, repr=False)
|
|
54
56
|
total_items_nb: int = field(init=False, repr=False)
|
|
55
57
|
sort_by_qs: str = field(init=False, repr=False)
|
|
58
|
+
raise_errors: Optional[bool] = field(init=False, repr=False)
|
eodag/plugins/search/base.py
CHANGED
|
@@ -29,6 +29,7 @@ from eodag.api.product.metadata_mapping import (
|
|
|
29
29
|
NOT_MAPPED,
|
|
30
30
|
mtd_cfg_as_conversion_and_querypath,
|
|
31
31
|
)
|
|
32
|
+
from eodag.api.search_result import SearchResult
|
|
32
33
|
from eodag.plugins.base import PluginTopic
|
|
33
34
|
from eodag.plugins.search import PreparedSearch
|
|
34
35
|
from eodag.types import model_fields_to_annotated
|
|
@@ -52,7 +53,6 @@ if TYPE_CHECKING:
|
|
|
52
53
|
from mypy_boto3_s3 import S3ServiceResource
|
|
53
54
|
from requests.auth import AuthBase
|
|
54
55
|
|
|
55
|
-
from eodag.api.product import EOProduct
|
|
56
56
|
from eodag.config import PluginConfig
|
|
57
57
|
|
|
58
58
|
logger = logging.getLogger("eodag.search.base")
|
|
@@ -97,7 +97,7 @@ class Search(PluginTopic):
|
|
|
97
97
|
self,
|
|
98
98
|
prep: PreparedSearch = PreparedSearch(),
|
|
99
99
|
**kwargs: Any,
|
|
100
|
-
) ->
|
|
100
|
+
) -> SearchResult:
|
|
101
101
|
"""Implementation of how the products must be searched goes here.
|
|
102
102
|
|
|
103
103
|
This method must return a tuple with (1) a list of :class:`~eodag.api.product._product.EOProduct` instances
|
|
@@ -403,10 +403,10 @@ class Search(PluginTopic):
|
|
|
403
403
|
return queryables
|
|
404
404
|
else:
|
|
405
405
|
all_queryables: dict[str, Any] = {}
|
|
406
|
-
for
|
|
407
|
-
self.config.collection_config = collection_configs[
|
|
408
|
-
|
|
409
|
-
all_queryables.update(
|
|
406
|
+
for col in available_collections:
|
|
407
|
+
self.config.collection_config = collection_configs[col]
|
|
408
|
+
col_queryables = self._get_collection_queryables(col, None, filters)
|
|
409
|
+
all_queryables.update(col_queryables)
|
|
410
410
|
# reset defaults because they may vary between collections
|
|
411
411
|
for k, v in all_queryables.items():
|
|
412
412
|
v.__metadata__[0].default = getattr(
|
|
@@ -31,10 +31,8 @@ import orjson
|
|
|
31
31
|
from dateutil.parser import isoparse
|
|
32
32
|
from dateutil.tz import tzutc
|
|
33
33
|
from dateutil.utils import today
|
|
34
|
-
from pydantic import Field
|
|
35
34
|
from pydantic.fields import FieldInfo
|
|
36
35
|
from requests.auth import AuthBase
|
|
37
|
-
from shapely.geometry.base import BaseGeometry
|
|
38
36
|
from typing_extensions import get_args # noqa: F401
|
|
39
37
|
|
|
40
38
|
from eodag.api.product import EOProduct
|
|
@@ -47,7 +45,7 @@ from eodag.api.product.metadata_mapping import (
|
|
|
47
45
|
mtd_cfg_as_conversion_and_querypath,
|
|
48
46
|
properties_from_json,
|
|
49
47
|
)
|
|
50
|
-
from eodag.api.search_result import RawSearchResult
|
|
48
|
+
from eodag.api.search_result import RawSearchResult, SearchResult
|
|
51
49
|
from eodag.plugins.search import PreparedSearch
|
|
52
50
|
from eodag.plugins.search.qssearch import PostJsonSearch, QueryStringSearch
|
|
53
51
|
from eodag.types import json_field_definition_to_python # noqa: F401
|
|
@@ -58,6 +56,8 @@ from eodag.utils import (
|
|
|
58
56
|
deepcopy,
|
|
59
57
|
dict_items_recursive_sort,
|
|
60
58
|
format_string,
|
|
59
|
+
get_geometry_from_ecmwf_area,
|
|
60
|
+
get_geometry_from_ecmwf_feature,
|
|
61
61
|
get_geometry_from_various,
|
|
62
62
|
)
|
|
63
63
|
from eodag.utils.cache import instance_cached_method
|
|
@@ -145,6 +145,7 @@ ECMWF_KEYWORDS = {
|
|
|
145
145
|
COP_DS_KEYWORDS = {
|
|
146
146
|
"aerosol_type",
|
|
147
147
|
"altitude",
|
|
148
|
+
"area",
|
|
148
149
|
"band",
|
|
149
150
|
"cdr_type",
|
|
150
151
|
"data_format",
|
|
@@ -190,6 +191,7 @@ COP_DS_KEYWORDS = {
|
|
|
190
191
|
"statistic",
|
|
191
192
|
"system_version",
|
|
192
193
|
"temporal_aggregation",
|
|
194
|
+
"temporal_resolution",
|
|
193
195
|
"time_aggregation",
|
|
194
196
|
"time_reference",
|
|
195
197
|
"time_step",
|
|
@@ -287,9 +289,22 @@ def _update_properties_from_element(
|
|
|
287
289
|
prop["description"] = description
|
|
288
290
|
|
|
289
291
|
|
|
290
|
-
def ecmwf_format(v: str) -> str:
|
|
291
|
-
"""Add ECMWF prefix to value v if v is a ECMWF keyword.
|
|
292
|
-
|
|
292
|
+
def ecmwf_format(v: str, alias: bool = True) -> str:
|
|
293
|
+
"""Add ECMWF prefix to value v if v is a ECMWF keyword.
|
|
294
|
+
|
|
295
|
+
:param v: parameter to format
|
|
296
|
+
:param alias: whether to format for alias (with ':') or for query param (False, with '_')
|
|
297
|
+
:return: formatted parameter
|
|
298
|
+
|
|
299
|
+
>>> ecmwf_format('dataset', alias=False)
|
|
300
|
+
'ecmwf_dataset'
|
|
301
|
+
>>> ecmwf_format('variable')
|
|
302
|
+
'ecmwf:variable'
|
|
303
|
+
>>> ecmwf_format('unknown_param')
|
|
304
|
+
'unknown_param'
|
|
305
|
+
"""
|
|
306
|
+
separator = ":" if alias else "_"
|
|
307
|
+
return f"{ECMWF_PREFIX[:-1]}{separator}{v}" if v in ALLOWED_KEYWORDS else v
|
|
293
308
|
|
|
294
309
|
|
|
295
310
|
def get_min_max(
|
|
@@ -447,6 +462,28 @@ class ECMWFSearch(PostJsonSearch):
|
|
|
447
462
|
queryables for a specific collection
|
|
448
463
|
* :attr:`~eodag.config.PluginConfig.DiscoverQueryables.constraints_url` (``str``): url of the constraint file
|
|
449
464
|
used to build queryables
|
|
465
|
+
|
|
466
|
+
* :attr:`~eodag.config.PluginConfig.dynamic_discover_queryables`
|
|
467
|
+
(``list`` [:class:`~eodag.config.PluginConfig.DynamicDiscoverQueryables`]): list of configurations to fetch
|
|
468
|
+
the queryables from different provider queryables endpoints. A configuration is used based on the given
|
|
469
|
+
selection criterias. The first match is used. If no match is found, it falls back to standard behaviors
|
|
470
|
+
(e.g. discovery using :attr:`~eodag.config.PluginConfig.discover_queryables`).
|
|
471
|
+
Each element of the list has the following keys:
|
|
472
|
+
|
|
473
|
+
* :attr:`~eodag.config.PluginConfig.DynamicDiscoverQueryables.collection_selector`
|
|
474
|
+
(``list`` [:class:`~eodag.config.PluginConfig.CollectionSelector`]): list of collection selection
|
|
475
|
+
criterias. The configuration given in
|
|
476
|
+
:attr:`~eodag.config.PluginConfig.DynamicDiscoverQueryables.discover_queryables` is used if any collection
|
|
477
|
+
selector matches the search parameters. The selector matches if the field value starts with the given
|
|
478
|
+
prefix, i.e. it matches if ``parameters[field].startswith(prefix)==True``. It has the following keys:
|
|
479
|
+
|
|
480
|
+
* :attr:`~eodag.config.PluginConfig.CollectionSelector.field` (``str``) Field in the search parameters to
|
|
481
|
+
match
|
|
482
|
+
* :attr:`~eodag.config.PluginConfig.CollectionSelector.prefix` (``str``) Prefix to match in the field
|
|
483
|
+
|
|
484
|
+
* :attr:`~eodag.config.PluginConfig.DynamicDiscoverQueryables.discover_queryables`
|
|
485
|
+
(``list`` [:class:`~eodag.config.PluginConfig.DiscoverQueryables`]): same as
|
|
486
|
+
:attr:`~eodag.config.PluginConfig.discover_queryables` above.
|
|
450
487
|
"""
|
|
451
488
|
|
|
452
489
|
def __init__(self, provider: str, config: PluginConfig) -> None:
|
|
@@ -480,7 +517,9 @@ class ECMWFSearch(PostJsonSearch):
|
|
|
480
517
|
},
|
|
481
518
|
)
|
|
482
519
|
|
|
483
|
-
def do_search(
|
|
520
|
+
def do_search(
|
|
521
|
+
self, prep: PreparedSearch = PreparedSearch(items_per_page=None), **kwargs: Any
|
|
522
|
+
) -> RawSearchResult:
|
|
484
523
|
"""Should perform the actual search request.
|
|
485
524
|
|
|
486
525
|
:param args: arguments to be used in the search
|
|
@@ -488,13 +527,21 @@ class ECMWFSearch(PostJsonSearch):
|
|
|
488
527
|
:return: list containing the results from the provider in json format
|
|
489
528
|
"""
|
|
490
529
|
# no real search. We fake it all
|
|
491
|
-
|
|
530
|
+
raw_search_results = RawSearchResult([{}])
|
|
531
|
+
raw_search_results.search_params = kwargs
|
|
532
|
+
raw_search_results.query_params = (
|
|
533
|
+
prep.query_params if hasattr(prep, "query_params") else {}
|
|
534
|
+
)
|
|
535
|
+
raw_search_results.collection_def_params = (
|
|
536
|
+
prep.collection_def_params if hasattr(prep, "collection_def_params") else {}
|
|
537
|
+
)
|
|
538
|
+
return raw_search_results
|
|
492
539
|
|
|
493
540
|
def query(
|
|
494
541
|
self,
|
|
495
542
|
prep: PreparedSearch = PreparedSearch(),
|
|
496
543
|
**kwargs: Any,
|
|
497
|
-
) ->
|
|
544
|
+
) -> SearchResult:
|
|
498
545
|
"""Build ready-to-download SearchResult
|
|
499
546
|
|
|
500
547
|
:param prep: :class:`~eodag.plugins.search.PreparedSearch` object containing information needed for the search
|
|
@@ -505,11 +552,11 @@ class ECMWFSearch(PostJsonSearch):
|
|
|
505
552
|
if not collection:
|
|
506
553
|
collection = kwargs.get("collection")
|
|
507
554
|
kwargs = self._preprocess_search_params(kwargs, collection)
|
|
508
|
-
result
|
|
509
|
-
if prep.count and not
|
|
510
|
-
|
|
555
|
+
result = super().query(prep, **kwargs)
|
|
556
|
+
if prep.count and not result.number_matched:
|
|
557
|
+
result.number_matched = 1
|
|
511
558
|
|
|
512
|
-
return result
|
|
559
|
+
return result
|
|
513
560
|
|
|
514
561
|
def clear(self) -> None:
|
|
515
562
|
"""Clear search context"""
|
|
@@ -570,7 +617,9 @@ class ECMWFSearch(PostJsonSearch):
|
|
|
570
617
|
params["geometry"] = _dc_qp["area"].split("/")
|
|
571
618
|
|
|
572
619
|
params = {
|
|
573
|
-
k.removeprefix(ECMWF_PREFIX)
|
|
620
|
+
k.removeprefix(ECMWF_PREFIX).removeprefix(f"{ECMWF_PREFIX[:-1]}_"): v
|
|
621
|
+
for k, v in params.items()
|
|
622
|
+
if v is not None
|
|
574
623
|
}
|
|
575
624
|
|
|
576
625
|
# dates
|
|
@@ -609,6 +658,14 @@ class ECMWFSearch(PostJsonSearch):
|
|
|
609
658
|
# geometry
|
|
610
659
|
if "geometry" in params:
|
|
611
660
|
params["geometry"] = get_geometry_from_various(geometry=params["geometry"])
|
|
661
|
+
# ECMWF Polytope uses non-geojson structure for features
|
|
662
|
+
if "feature" in params:
|
|
663
|
+
params["geometry"] = get_geometry_from_ecmwf_feature(params["feature"])
|
|
664
|
+
params.pop("feature")
|
|
665
|
+
# bounding box in area format
|
|
666
|
+
if "area" in params:
|
|
667
|
+
params["geometry"] = get_geometry_from_ecmwf_area(params["area"])
|
|
668
|
+
params.pop("area")
|
|
612
669
|
|
|
613
670
|
return params
|
|
614
671
|
|
|
@@ -617,7 +674,7 @@ class ECMWFSearch(PostJsonSearch):
|
|
|
617
674
|
) -> None:
|
|
618
675
|
"""checks if start and end date are present in the keywords and adds them if not"""
|
|
619
676
|
|
|
620
|
-
if START and END in keywords:
|
|
677
|
+
if START in keywords and END in keywords:
|
|
621
678
|
return
|
|
622
679
|
|
|
623
680
|
collection_conf = getattr(self.config, "metadata_mapping", {})
|
|
@@ -666,14 +723,32 @@ class ECMWFSearch(PostJsonSearch):
|
|
|
666
723
|
getattr(self.config, "products", {}).get(collection, {})
|
|
667
724
|
)
|
|
668
725
|
default_values.pop("metadata_mapping", None)
|
|
726
|
+
default_values.pop("metadata_mapping_from_product", None)
|
|
669
727
|
|
|
670
728
|
filters["collection"] = collection
|
|
671
729
|
queryables = self.discover_queryables(**{**default_values, **filters}) or {}
|
|
672
730
|
|
|
673
731
|
return QueryablesDict(additional_properties=False, **queryables)
|
|
674
732
|
|
|
733
|
+
def _find_dynamic_queryables_config(
|
|
734
|
+
self, kwargs: dict[str, Any], dynamic_config: list
|
|
735
|
+
) -> dict[str, Any]:
|
|
736
|
+
"""Find the appropriate queryables configuration from dynamic configuration.
|
|
737
|
+
|
|
738
|
+
:param kwargs: Search parameters
|
|
739
|
+
:param dynamic_config: List of dynamic discover queryables configurations
|
|
740
|
+
:return: Found queryables configuration or empty dict
|
|
741
|
+
"""
|
|
742
|
+
for dc in dynamic_config:
|
|
743
|
+
for cs in dc["collection_selector"]:
|
|
744
|
+
field = cs["field"]
|
|
745
|
+
if kwargs[field].startswith(cs["prefix"]):
|
|
746
|
+
return dc["discover_queryables"]
|
|
747
|
+
return {}
|
|
748
|
+
|
|
675
749
|
def discover_queryables(
|
|
676
|
-
self,
|
|
750
|
+
self,
|
|
751
|
+
**kwargs: Any,
|
|
677
752
|
) -> Optional[dict[str, Annotated[Any, FieldInfo]]]:
|
|
678
753
|
"""Fetch queryables list from provider using its constraints file
|
|
679
754
|
|
|
@@ -683,10 +758,13 @@ class ECMWFSearch(PostJsonSearch):
|
|
|
683
758
|
"""
|
|
684
759
|
collection = kwargs.pop("collection")
|
|
685
760
|
|
|
686
|
-
|
|
761
|
+
col_config = self.get_collection_def_params(collection)
|
|
687
762
|
|
|
688
|
-
default_values = deepcopy(
|
|
763
|
+
default_values = deepcopy(col_config)
|
|
689
764
|
default_values.pop("metadata_mapping", None)
|
|
765
|
+
default_values.pop("metadata_mapping_from_product", None)
|
|
766
|
+
default_values.pop("discover_queryables", None)
|
|
767
|
+
kwargs.pop("discover_queryables", None)
|
|
690
768
|
filters = {**default_values, **kwargs}
|
|
691
769
|
|
|
692
770
|
if "start" in filters:
|
|
@@ -694,34 +772,40 @@ class ECMWFSearch(PostJsonSearch):
|
|
|
694
772
|
if "end" in filters:
|
|
695
773
|
filters[END] = filters.pop("end")
|
|
696
774
|
|
|
697
|
-
# extract default datetime
|
|
698
|
-
|
|
699
|
-
|
|
700
|
-
|
|
775
|
+
# extract default datetime and convert geometry
|
|
776
|
+
try:
|
|
777
|
+
processed_filters = self._preprocess_search_params(
|
|
778
|
+
deepcopy(filters), collection
|
|
779
|
+
)
|
|
780
|
+
except Exception as e:
|
|
781
|
+
raise ValidationError(e.args[0]) from e
|
|
782
|
+
|
|
783
|
+
# dynamic_discover_queryables for WekeoECMWFSearch
|
|
784
|
+
queryables_config = {}
|
|
785
|
+
if dynamic_config := getattr(self.config, "dynamic_discover_queryables", []):
|
|
786
|
+
queryables_config = self._find_dynamic_queryables_config(
|
|
787
|
+
kwargs, dynamic_config
|
|
788
|
+
)
|
|
701
789
|
|
|
702
|
-
|
|
703
|
-
|
|
704
|
-
|
|
705
|
-
)
|
|
790
|
+
provider_dq = getattr(self.config, "discover_queryables", {}) or {}
|
|
791
|
+
product_dq = col_config.get("discover_queryables", {}) or {}
|
|
792
|
+
dq_conf = {**provider_dq, **product_dq, **queryables_config}
|
|
793
|
+
constraints_url = format_metadata(dq_conf.get("constraints_url", ""), **filters)
|
|
706
794
|
constraints: list[dict[str, Any]] = self._fetch_data(constraints_url)
|
|
707
795
|
|
|
708
|
-
form_url = format_metadata(
|
|
709
|
-
getattr(self.config, "discover_queryables", {}).get("form_url", ""),
|
|
710
|
-
**filters,
|
|
711
|
-
)
|
|
796
|
+
form_url = format_metadata(dq_conf.get("form_url", ""), **filters)
|
|
712
797
|
form: list[dict[str, Any]] = self._fetch_data(form_url)
|
|
713
798
|
|
|
714
799
|
formated_filters = self.format_as_provider_keyword(
|
|
715
|
-
collection, processed_filters
|
|
800
|
+
collection, deepcopy(processed_filters)
|
|
716
801
|
)
|
|
717
802
|
# we re-apply kwargs input to consider override of year, month, day and time.
|
|
718
803
|
for k, v in {**default_values, **kwargs}.items():
|
|
719
|
-
key = k.removeprefix(ECMWF_PREFIX)
|
|
804
|
+
key = k.removeprefix(ECMWF_PREFIX).removeprefix(f"{ECMWF_PREFIX[:-1]}_")
|
|
720
805
|
|
|
721
806
|
if key not in ALLOWED_KEYWORDS | {
|
|
722
807
|
START,
|
|
723
808
|
END,
|
|
724
|
-
"geom",
|
|
725
809
|
"geometry",
|
|
726
810
|
}:
|
|
727
811
|
raise ValidationError(
|
|
@@ -770,19 +854,20 @@ class ECMWFSearch(PostJsonSearch):
|
|
|
770
854
|
# To check if all keywords are queryable parameters, we check if they are in the
|
|
771
855
|
# available values or the collection config (available values calculated from the
|
|
772
856
|
# constraints might not include all queryables)
|
|
773
|
-
for keyword in
|
|
857
|
+
for keyword in processed_filters:
|
|
774
858
|
if (
|
|
775
859
|
keyword
|
|
776
860
|
not in available_values.keys()
|
|
777
|
-
|
|
|
861
|
+
| col_config.keys()
|
|
778
862
|
| {
|
|
779
863
|
START,
|
|
780
864
|
END,
|
|
781
|
-
"geom",
|
|
782
865
|
"geometry",
|
|
783
866
|
}
|
|
784
867
|
and keyword not in [f["name"] for f in form]
|
|
785
|
-
and keyword.removeprefix(ECMWF_PREFIX)
|
|
868
|
+
and keyword.removeprefix(ECMWF_PREFIX).removeprefix(
|
|
869
|
+
f"{ECMWF_PREFIX[:-1]}_"
|
|
870
|
+
)
|
|
786
871
|
not in set(list(available_values.keys()) + [f["name"] for f in form])
|
|
787
872
|
):
|
|
788
873
|
raise ValidationError(
|
|
@@ -803,10 +888,11 @@ class ECMWFSearch(PostJsonSearch):
|
|
|
803
888
|
|
|
804
889
|
# ecmwf:date is replaced by start and end.
|
|
805
890
|
# start and end filters are supported whenever combinations of "year", "month", "day" filters exist
|
|
891
|
+
queryable_prefix = f"{ECMWF_PREFIX[:-1]}_"
|
|
806
892
|
if (
|
|
807
|
-
queryables.pop(f"{
|
|
808
|
-
or f"{
|
|
809
|
-
or f"{
|
|
893
|
+
queryables.pop(f"{queryable_prefix}date", None)
|
|
894
|
+
or f"{queryable_prefix}year" in queryables
|
|
895
|
+
or f"{queryable_prefix}hyear" in queryables
|
|
810
896
|
):
|
|
811
897
|
queryables.update(
|
|
812
898
|
{
|
|
@@ -822,13 +908,7 @@ class ECMWFSearch(PostJsonSearch):
|
|
|
822
908
|
|
|
823
909
|
# area is geom in EODAG.
|
|
824
910
|
if queryables.pop("area", None):
|
|
825
|
-
queryables["geom"] =
|
|
826
|
-
Union[str, dict[str, float], BaseGeometry],
|
|
827
|
-
Field(
|
|
828
|
-
None,
|
|
829
|
-
description="Read EODAG documentation for all supported geometry format.",
|
|
830
|
-
),
|
|
831
|
-
]
|
|
911
|
+
queryables["geom"] = Queryables.get_with_default("geom", None)
|
|
832
912
|
|
|
833
913
|
return queryables
|
|
834
914
|
|
|
@@ -1026,12 +1106,15 @@ class ECMWFSearch(PostJsonSearch):
|
|
|
1026
1106
|
if is_required:
|
|
1027
1107
|
required_list.append(name)
|
|
1028
1108
|
|
|
1029
|
-
|
|
1109
|
+
formatted_param = ecmwf_format(name, alias=False)
|
|
1110
|
+
formatted_alias = ecmwf_format(name)
|
|
1111
|
+
queryables[formatted_param] = Annotated[
|
|
1030
1112
|
get_args(
|
|
1031
1113
|
json_field_definition_to_python(
|
|
1032
1114
|
prop,
|
|
1033
1115
|
default_value=default,
|
|
1034
1116
|
required=is_required,
|
|
1117
|
+
alias=formatted_alias,
|
|
1035
1118
|
)
|
|
1036
1119
|
)
|
|
1037
1120
|
]
|
|
@@ -1061,14 +1144,16 @@ class ECMWFSearch(PostJsonSearch):
|
|
|
1061
1144
|
for name, values in available_values.items():
|
|
1062
1145
|
# Rename keywords from form with metadata mapping.
|
|
1063
1146
|
# Needed to map constraints like "xxxx" to eodag parameter "ecmwf:xxxx"
|
|
1064
|
-
|
|
1147
|
+
formatted_param = ecmwf_format(name, alias=False)
|
|
1148
|
+
formatted_alias = ecmwf_format(name)
|
|
1065
1149
|
|
|
1066
|
-
queryables[
|
|
1150
|
+
queryables[formatted_param] = Annotated[
|
|
1067
1151
|
get_args(
|
|
1068
1152
|
json_field_definition_to_python(
|
|
1069
1153
|
{"type": "string", "title": name, "enum": values},
|
|
1070
1154
|
default_value=defaults.get(name),
|
|
1071
|
-
required=bool(
|
|
1155
|
+
required=bool(formatted_alias in required),
|
|
1156
|
+
alias=formatted_alias,
|
|
1072
1157
|
)
|
|
1073
1158
|
)
|
|
1074
1159
|
]
|
|
@@ -1346,7 +1431,7 @@ class MeteoblueSearch(ECMWFSearch):
|
|
|
1346
1431
|
|
|
1347
1432
|
def do_search(
|
|
1348
1433
|
self, prep: PreparedSearch = PreparedSearch(items_per_page=None), **kwargs: Any
|
|
1349
|
-
) ->
|
|
1434
|
+
) -> RawSearchResult:
|
|
1350
1435
|
"""Perform the actual search request, and return result in a single element.
|
|
1351
1436
|
|
|
1352
1437
|
:param prep: :class:`~eodag.plugins.search.PreparedSearch` object containing information for the search
|
|
@@ -1361,8 +1446,12 @@ class MeteoblueSearch(ECMWFSearch):
|
|
|
1361
1446
|
f" {self.__class__.__name__} instance"
|
|
1362
1447
|
)
|
|
1363
1448
|
response = self._request(prep)
|
|
1449
|
+
raw_search_results = RawSearchResult([response.json()])
|
|
1450
|
+
raw_search_results.search_params = kwargs
|
|
1364
1451
|
|
|
1365
|
-
|
|
1452
|
+
raw_search_results.query_params = prep.query_params
|
|
1453
|
+
raw_search_results.collection_def_params = prep.collection_def_params
|
|
1454
|
+
return raw_search_results
|
|
1366
1455
|
|
|
1367
1456
|
def build_query_string(
|
|
1368
1457
|
self, collection: str, query_dict: dict[str, Any]
|
|
@@ -1539,7 +1628,9 @@ class WekeoECMWFSearch(ECMWFSearch):
|
|
|
1539
1628
|
|
|
1540
1629
|
return normalized
|
|
1541
1630
|
|
|
1542
|
-
def do_search(
|
|
1631
|
+
def do_search(
|
|
1632
|
+
self, prep: PreparedSearch = PreparedSearch(items_per_page=None), **kwargs: Any
|
|
1633
|
+
) -> RawSearchResult:
|
|
1543
1634
|
"""Should perform the actual search request.
|
|
1544
1635
|
|
|
1545
1636
|
:param args: arguments to be used in the search
|
|
@@ -1549,6 +1640,16 @@ class WekeoECMWFSearch(ECMWFSearch):
|
|
|
1549
1640
|
if "id" in kwargs and "ORDERABLE" not in kwargs["id"]:
|
|
1550
1641
|
# id is order id (only letters and numbers) -> use parent normalize results.
|
|
1551
1642
|
# No real search. We fake it all, then check order status using given id
|
|
1552
|
-
|
|
1643
|
+
raw_search_results = RawSearchResult([{}])
|
|
1644
|
+
raw_search_results.search_params = kwargs
|
|
1645
|
+
raw_search_results.query_params = (
|
|
1646
|
+
prep.query_params if hasattr(prep, "query_params") else {}
|
|
1647
|
+
)
|
|
1648
|
+
raw_search_results.collection_def_params = (
|
|
1649
|
+
prep.collection_def_params
|
|
1650
|
+
if hasattr(prep, "collection_def_params")
|
|
1651
|
+
else {}
|
|
1652
|
+
)
|
|
1653
|
+
return raw_search_results
|
|
1553
1654
|
else:
|
|
1554
|
-
return QueryStringSearch.do_search(self,
|
|
1655
|
+
return QueryStringSearch.do_search(self, prep, **kwargs)
|