eodag 3.10.1__py3-none-any.whl → 4.0.0a2__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- eodag/__init__.py +6 -1
- eodag/api/collection.py +353 -0
- eodag/api/core.py +606 -641
- eodag/api/product/__init__.py +3 -3
- eodag/api/product/_product.py +74 -56
- eodag/api/product/drivers/__init__.py +4 -46
- eodag/api/product/drivers/base.py +0 -28
- eodag/api/product/metadata_mapping.py +178 -216
- eodag/api/search_result.py +156 -15
- eodag/cli.py +83 -403
- eodag/config.py +81 -51
- eodag/plugins/apis/base.py +2 -2
- eodag/plugins/apis/ecmwf.py +36 -25
- eodag/plugins/apis/usgs.py +55 -40
- eodag/plugins/authentication/base.py +1 -3
- eodag/plugins/crunch/filter_date.py +3 -3
- eodag/plugins/crunch/filter_latest_intersect.py +2 -2
- eodag/plugins/crunch/filter_latest_tpl_name.py +1 -1
- eodag/plugins/download/aws.py +46 -42
- eodag/plugins/download/base.py +13 -14
- eodag/plugins/download/http.py +65 -65
- eodag/plugins/manager.py +28 -29
- eodag/plugins/search/__init__.py +6 -4
- eodag/plugins/search/base.py +131 -80
- eodag/plugins/search/build_search_result.py +245 -173
- eodag/plugins/search/cop_marine.py +87 -56
- eodag/plugins/search/csw.py +47 -37
- eodag/plugins/search/qssearch.py +653 -429
- eodag/plugins/search/stac_list_assets.py +1 -1
- eodag/plugins/search/static_stac_search.py +43 -44
- eodag/resources/{product_types.yml → collections.yml} +2594 -2453
- eodag/resources/ext_collections.json +1 -1
- eodag/resources/ext_product_types.json +1 -1
- eodag/resources/providers.yml +2706 -2733
- eodag/resources/stac_provider.yml +50 -92
- eodag/resources/user_conf_template.yml +9 -0
- eodag/types/__init__.py +2 -0
- eodag/types/queryables.py +70 -91
- eodag/types/search_args.py +1 -1
- eodag/utils/__init__.py +97 -21
- eodag/utils/dates.py +0 -12
- eodag/utils/exceptions.py +6 -6
- eodag/utils/free_text_search.py +3 -3
- eodag/utils/repr.py +2 -0
- {eodag-3.10.1.dist-info → eodag-4.0.0a2.dist-info}/METADATA +13 -99
- eodag-4.0.0a2.dist-info/RECORD +93 -0
- {eodag-3.10.1.dist-info → eodag-4.0.0a2.dist-info}/entry_points.txt +0 -4
- eodag/plugins/authentication/oauth.py +0 -60
- eodag/plugins/download/creodias_s3.py +0 -71
- eodag/plugins/download/s3rest.py +0 -351
- eodag/plugins/search/data_request_search.py +0 -565
- eodag/resources/stac.yml +0 -294
- eodag/resources/stac_api.yml +0 -2105
- eodag/rest/__init__.py +0 -24
- eodag/rest/cache.py +0 -70
- eodag/rest/config.py +0 -67
- eodag/rest/constants.py +0 -26
- eodag/rest/core.py +0 -764
- eodag/rest/errors.py +0 -210
- eodag/rest/server.py +0 -604
- eodag/rest/server.wsgi +0 -6
- eodag/rest/stac.py +0 -1032
- eodag/rest/templates/README +0 -1
- eodag/rest/types/__init__.py +0 -18
- eodag/rest/types/collections_search.py +0 -44
- eodag/rest/types/eodag_search.py +0 -386
- eodag/rest/types/queryables.py +0 -174
- eodag/rest/types/stac_search.py +0 -272
- eodag/rest/utils/__init__.py +0 -207
- eodag/rest/utils/cql_evaluate.py +0 -119
- eodag/rest/utils/rfc3339.py +0 -64
- eodag-3.10.1.dist-info/RECORD +0 -116
- {eodag-3.10.1.dist-info → eodag-4.0.0a2.dist-info}/WHEEL +0 -0
- {eodag-3.10.1.dist-info → eodag-4.0.0a2.dist-info}/licenses/LICENSE +0 -0
- {eodag-3.10.1.dist-info → eodag-4.0.0a2.dist-info}/top_level.txt +0 -0
|
@@ -31,10 +31,8 @@ import orjson
|
|
|
31
31
|
from dateutil.parser import isoparse
|
|
32
32
|
from dateutil.tz import tzutc
|
|
33
33
|
from dateutil.utils import today
|
|
34
|
-
from pydantic import Field
|
|
35
34
|
from pydantic.fields import FieldInfo
|
|
36
35
|
from requests.auth import AuthBase
|
|
37
|
-
from shapely.geometry.base import BaseGeometry
|
|
38
36
|
from typing_extensions import get_args # noqa: F401
|
|
39
37
|
|
|
40
38
|
from eodag.api.product import EOProduct
|
|
@@ -47,7 +45,7 @@ from eodag.api.product.metadata_mapping import (
|
|
|
47
45
|
mtd_cfg_as_conversion_and_querypath,
|
|
48
46
|
properties_from_json,
|
|
49
47
|
)
|
|
50
|
-
from eodag.api.search_result import RawSearchResult
|
|
48
|
+
from eodag.api.search_result import RawSearchResult, SearchResult
|
|
51
49
|
from eodag.plugins.search import PreparedSearch
|
|
52
50
|
from eodag.plugins.search.qssearch import PostJsonSearch, QueryStringSearch
|
|
53
51
|
from eodag.types import json_field_definition_to_python # noqa: F401
|
|
@@ -57,16 +55,13 @@ from eodag.utils import (
|
|
|
57
55
|
DEFAULT_SEARCH_TIMEOUT,
|
|
58
56
|
deepcopy,
|
|
59
57
|
dict_items_recursive_sort,
|
|
58
|
+
format_string,
|
|
60
59
|
get_geometry_from_ecmwf_area,
|
|
61
60
|
get_geometry_from_ecmwf_feature,
|
|
62
61
|
get_geometry_from_various,
|
|
63
62
|
)
|
|
64
63
|
from eodag.utils.cache import instance_cached_method
|
|
65
|
-
from eodag.utils.dates import
|
|
66
|
-
COMPACT_DATE_RANGE_PATTERN,
|
|
67
|
-
DATE_RANGE_PATTERN,
|
|
68
|
-
is_range_in_range,
|
|
69
|
-
)
|
|
64
|
+
from eodag.utils.dates import is_range_in_range
|
|
70
65
|
from eodag.utils.exceptions import DownloadError, NotAvailableError, ValidationError
|
|
71
66
|
from eodag.utils.requests import fetch_json
|
|
72
67
|
|
|
@@ -150,7 +145,6 @@ ECMWF_KEYWORDS = {
|
|
|
150
145
|
COP_DS_KEYWORDS = {
|
|
151
146
|
"aerosol_type",
|
|
152
147
|
"altitude",
|
|
153
|
-
"product_type",
|
|
154
148
|
"area",
|
|
155
149
|
"band",
|
|
156
150
|
"cdr_type",
|
|
@@ -183,13 +177,13 @@ COP_DS_KEYWORDS = {
|
|
|
183
177
|
"pressure_level",
|
|
184
178
|
"processing_level",
|
|
185
179
|
"processing_type",
|
|
180
|
+
"product_type",
|
|
186
181
|
"product_version",
|
|
187
182
|
"quantity",
|
|
188
183
|
"rcm",
|
|
189
184
|
"region",
|
|
190
185
|
"release_version",
|
|
191
186
|
"satellite",
|
|
192
|
-
"satellite_mission",
|
|
193
187
|
"sensor",
|
|
194
188
|
"sensor_and_algorithm",
|
|
195
189
|
"soil_level",
|
|
@@ -209,9 +203,9 @@ COP_DS_KEYWORDS = {
|
|
|
209
203
|
|
|
210
204
|
ALLOWED_KEYWORDS = ECMWF_KEYWORDS | COP_DS_KEYWORDS
|
|
211
205
|
|
|
212
|
-
END = "
|
|
206
|
+
END = "end_datetime"
|
|
213
207
|
|
|
214
|
-
START = "
|
|
208
|
+
START = "start_datetime"
|
|
215
209
|
|
|
216
210
|
|
|
217
211
|
def ecmwf_mtd() -> dict[str, Any]:
|
|
@@ -295,9 +289,22 @@ def _update_properties_from_element(
|
|
|
295
289
|
prop["description"] = description
|
|
296
290
|
|
|
297
291
|
|
|
298
|
-
def ecmwf_format(v: str) -> str:
|
|
299
|
-
"""Add ECMWF prefix to value v if v is a ECMWF keyword.
|
|
300
|
-
|
|
292
|
+
def ecmwf_format(v: str, alias: bool = True) -> str:
|
|
293
|
+
"""Add ECMWF prefix to value v if v is a ECMWF keyword.
|
|
294
|
+
|
|
295
|
+
:param v: parameter to format
|
|
296
|
+
:param alias: whether to format for alias (with ':') or for query param (False, with '_')
|
|
297
|
+
:return: formatted parameter
|
|
298
|
+
|
|
299
|
+
>>> ecmwf_format('dataset', alias=False)
|
|
300
|
+
'ecmwf_dataset'
|
|
301
|
+
>>> ecmwf_format('variable')
|
|
302
|
+
'ecmwf:variable'
|
|
303
|
+
>>> ecmwf_format('unknown_param')
|
|
304
|
+
'unknown_param'
|
|
305
|
+
"""
|
|
306
|
+
separator = ":" if alias else "_"
|
|
307
|
+
return f"{ECMWF_PREFIX[:-1]}{separator}{v}" if v in ALLOWED_KEYWORDS else v
|
|
301
308
|
|
|
302
309
|
|
|
303
310
|
def get_min_max(
|
|
@@ -314,11 +321,11 @@ def append_time(input_date: date, time: Optional[str]) -> datetime:
|
|
|
314
321
|
"""
|
|
315
322
|
Parses a time string in format HHMM and appends it to a date.
|
|
316
323
|
|
|
317
|
-
if the time string is in format HH:MM we convert it to HHMM
|
|
324
|
+
if the time string is in format HH:MM or HH_MM we convert it to HHMM
|
|
318
325
|
"""
|
|
319
326
|
if not time:
|
|
320
327
|
time = "0000"
|
|
321
|
-
time =
|
|
328
|
+
time = re.sub(":|_", "", time)
|
|
322
329
|
if time == "2400":
|
|
323
330
|
time = "0000"
|
|
324
331
|
dt = datetime.combine(input_date, datetime.strptime(time, "%H%M").time())
|
|
@@ -444,16 +451,39 @@ class ECMWFSearch(PostJsonSearch):
|
|
|
444
451
|
used to parse metadata but that must not be included to the query
|
|
445
452
|
* :attr:`~eodag.config.PluginConfig.end_date_excluded` (``bool``): Set to `False` if
|
|
446
453
|
provider does not include end date to search
|
|
454
|
+
* :attr:`~eodag.config.PluginConfig.dates_required` (``bool``): if date parameters are mandatory in the request
|
|
447
455
|
* :attr:`~eodag.config.PluginConfig.discover_queryables`
|
|
448
456
|
(:class:`~eodag.config.PluginConfig.DiscoverQueryables`): configuration to fetch the queryables from a
|
|
449
457
|
provider queryables endpoint; It has the following keys:
|
|
450
458
|
|
|
451
459
|
* :attr:`~eodag.config.PluginConfig.DiscoverQueryables.fetch_url` (``str``): url to fetch the queryables valid
|
|
452
|
-
for all
|
|
453
|
-
* :attr:`~eodag.config.PluginConfig.DiscoverQueryables.
|
|
454
|
-
queryables for a specific
|
|
460
|
+
for all collections
|
|
461
|
+
* :attr:`~eodag.config.PluginConfig.DiscoverQueryables.collection_fetch_url` (``str``): url to fetch the
|
|
462
|
+
queryables for a specific collection
|
|
455
463
|
* :attr:`~eodag.config.PluginConfig.DiscoverQueryables.constraints_url` (``str``): url of the constraint file
|
|
456
464
|
used to build queryables
|
|
465
|
+
|
|
466
|
+
* :attr:`~eodag.config.PluginConfig.dynamic_discover_queryables`
|
|
467
|
+
(``list`` [:class:`~eodag.config.PluginConfig.DynamicDiscoverQueryables`]): list of configurations to fetch
|
|
468
|
+
the queryables from different provider queryables endpoints. A configuration is used based on the given
|
|
469
|
+
selection criterias. The first match is used. If no match is found, it falls back to standard behaviors
|
|
470
|
+
(e.g. discovery using :attr:`~eodag.config.PluginConfig.discover_queryables`).
|
|
471
|
+
Each element of the list has the following keys:
|
|
472
|
+
|
|
473
|
+
* :attr:`~eodag.config.PluginConfig.DynamicDiscoverQueryables.collection_selector`
|
|
474
|
+
(``list`` [:class:`~eodag.config.PluginConfig.CollectionSelector`]): list of collection selection
|
|
475
|
+
criterias. The configuration given in
|
|
476
|
+
:attr:`~eodag.config.PluginConfig.DynamicDiscoverQueryables.discover_queryables` is used if any collection
|
|
477
|
+
selector matches the search parameters. The selector matches if the field value starts with the given
|
|
478
|
+
prefix, i.e. it matches if ``parameters[field].startswith(prefix)==True``. It has the following keys:
|
|
479
|
+
|
|
480
|
+
* :attr:`~eodag.config.PluginConfig.CollectionSelector.field` (``str``) Field in the search parameters to
|
|
481
|
+
match
|
|
482
|
+
* :attr:`~eodag.config.PluginConfig.CollectionSelector.prefix` (``str``) Prefix to match in the field
|
|
483
|
+
|
|
484
|
+
* :attr:`~eodag.config.PluginConfig.DynamicDiscoverQueryables.discover_queryables`
|
|
485
|
+
(``list`` [:class:`~eodag.config.PluginConfig.DiscoverQueryables`]): same as
|
|
486
|
+
:attr:`~eodag.config.PluginConfig.discover_queryables` above.
|
|
457
487
|
"""
|
|
458
488
|
|
|
459
489
|
def __init__(self, provider: str, config: PluginConfig) -> None:
|
|
@@ -462,10 +492,10 @@ class ECMWFSearch(PostJsonSearch):
|
|
|
462
492
|
**{
|
|
463
493
|
"id": "$.id",
|
|
464
494
|
"title": "$.id",
|
|
465
|
-
"
|
|
466
|
-
"
|
|
495
|
+
"order:status": OFFLINE_STATUS,
|
|
496
|
+
"eodag:download_link": "$.null",
|
|
467
497
|
"geometry": ["feature", "$.geometry"],
|
|
468
|
-
"
|
|
498
|
+
"eodag:default_geometry": "POLYGON((180 -90, 180 90, -180 90, -180 -90, 180 -90))",
|
|
469
499
|
},
|
|
470
500
|
**config.metadata_mapping,
|
|
471
501
|
}
|
|
@@ -487,7 +517,9 @@ class ECMWFSearch(PostJsonSearch):
|
|
|
487
517
|
},
|
|
488
518
|
)
|
|
489
519
|
|
|
490
|
-
def do_search(
|
|
520
|
+
def do_search(
|
|
521
|
+
self, prep: PreparedSearch = PreparedSearch(items_per_page=None), **kwargs: Any
|
|
522
|
+
) -> RawSearchResult:
|
|
491
523
|
"""Should perform the actual search request.
|
|
492
524
|
|
|
493
525
|
:param args: arguments to be used in the search
|
|
@@ -495,39 +527,47 @@ class ECMWFSearch(PostJsonSearch):
|
|
|
495
527
|
:return: list containing the results from the provider in json format
|
|
496
528
|
"""
|
|
497
529
|
# no real search. We fake it all
|
|
498
|
-
|
|
530
|
+
raw_search_results = RawSearchResult([{}])
|
|
531
|
+
raw_search_results.search_params = kwargs
|
|
532
|
+
raw_search_results.query_params = (
|
|
533
|
+
prep.query_params if hasattr(prep, "query_params") else {}
|
|
534
|
+
)
|
|
535
|
+
raw_search_results.collection_def_params = (
|
|
536
|
+
prep.collection_def_params if hasattr(prep, "collection_def_params") else {}
|
|
537
|
+
)
|
|
538
|
+
return raw_search_results
|
|
499
539
|
|
|
500
540
|
def query(
|
|
501
541
|
self,
|
|
502
542
|
prep: PreparedSearch = PreparedSearch(),
|
|
503
543
|
**kwargs: Any,
|
|
504
|
-
) ->
|
|
544
|
+
) -> SearchResult:
|
|
505
545
|
"""Build ready-to-download SearchResult
|
|
506
546
|
|
|
507
547
|
:param prep: :class:`~eodag.plugins.search.PreparedSearch` object containing information needed for the search
|
|
508
548
|
:param kwargs: keyword arguments to be used in the search
|
|
509
549
|
:returns: list of products and number of products (optional)
|
|
510
550
|
"""
|
|
511
|
-
|
|
512
|
-
if not
|
|
513
|
-
|
|
514
|
-
kwargs = self._preprocess_search_params(kwargs,
|
|
515
|
-
result
|
|
516
|
-
if prep.count and not
|
|
517
|
-
|
|
551
|
+
collection = prep.collection
|
|
552
|
+
if not collection:
|
|
553
|
+
collection = kwargs.get("collection")
|
|
554
|
+
kwargs = self._preprocess_search_params(kwargs, collection)
|
|
555
|
+
result = super().query(prep, **kwargs)
|
|
556
|
+
if prep.count and not result.number_matched:
|
|
557
|
+
result.number_matched = 1
|
|
518
558
|
|
|
519
|
-
return result
|
|
559
|
+
return result
|
|
520
560
|
|
|
521
561
|
def clear(self) -> None:
|
|
522
562
|
"""Clear search context"""
|
|
523
563
|
super().clear()
|
|
524
564
|
|
|
525
565
|
def build_query_string(
|
|
526
|
-
self,
|
|
566
|
+
self, collection: str, query_dict: dict[str, Any]
|
|
527
567
|
) -> tuple[dict[str, Any], str]:
|
|
528
568
|
"""Build The query string using the search parameters
|
|
529
569
|
|
|
530
|
-
:param
|
|
570
|
+
:param collection: collection id
|
|
531
571
|
:param query_dict: keyword arguments to be used in the query string
|
|
532
572
|
:return: formatted query params and encode query string
|
|
533
573
|
"""
|
|
@@ -542,21 +582,21 @@ class ECMWFSearch(PostJsonSearch):
|
|
|
542
582
|
ordered_kwargs.update(query_dict)
|
|
543
583
|
|
|
544
584
|
return super().build_query_string(
|
|
545
|
-
|
|
585
|
+
collection=collection, query_dict=ordered_kwargs
|
|
546
586
|
)
|
|
547
587
|
|
|
548
588
|
def _preprocess_search_params(
|
|
549
|
-
self, params: dict[str, Any],
|
|
589
|
+
self, params: dict[str, Any], collection: Optional[str]
|
|
550
590
|
) -> dict[str, Any]:
|
|
551
591
|
"""Preprocess search parameters before making a request to the CDS API.
|
|
552
592
|
|
|
553
593
|
This method is responsible for checking and updating the provided search parameters
|
|
554
|
-
to ensure that required parameters like '
|
|
555
|
-
'
|
|
594
|
+
to ensure that required parameters like 'collection', 'start_datetime',
|
|
595
|
+
'end_datetime', and 'geometry' are properly set. If not specified
|
|
556
596
|
in the input parameters, default values or values from the configuration are used.
|
|
557
597
|
|
|
558
598
|
:param params: Search parameters to be preprocessed.
|
|
559
|
-
:param
|
|
599
|
+
:param collection: (optional) collection id
|
|
560
600
|
"""
|
|
561
601
|
|
|
562
602
|
_dc_qs = params.get("_dc_qs")
|
|
@@ -577,13 +617,15 @@ class ECMWFSearch(PostJsonSearch):
|
|
|
577
617
|
params["geometry"] = _dc_qp["area"].split("/")
|
|
578
618
|
|
|
579
619
|
params = {
|
|
580
|
-
k.removeprefix(ECMWF_PREFIX)
|
|
620
|
+
k.removeprefix(ECMWF_PREFIX).removeprefix(f"{ECMWF_PREFIX[:-1]}_"): v
|
|
621
|
+
for k, v in params.items()
|
|
622
|
+
if v is not None
|
|
581
623
|
}
|
|
582
624
|
|
|
583
625
|
# dates
|
|
584
626
|
# check if default dates have to be added
|
|
585
627
|
if getattr(self.config, "dates_required", False):
|
|
586
|
-
self._check_date_params(params,
|
|
628
|
+
self._check_date_params(params, collection)
|
|
587
629
|
|
|
588
630
|
# adapt end date if it is midnight
|
|
589
631
|
if END in params:
|
|
@@ -628,84 +670,101 @@ class ECMWFSearch(PostJsonSearch):
|
|
|
628
670
|
return params
|
|
629
671
|
|
|
630
672
|
def _check_date_params(
|
|
631
|
-
self, keywords: dict[str, Any],
|
|
673
|
+
self, keywords: dict[str, Any], collection: Optional[str]
|
|
632
674
|
) -> None:
|
|
633
675
|
"""checks if start and end date are present in the keywords and adds them if not"""
|
|
634
676
|
|
|
635
|
-
if START and END in keywords:
|
|
677
|
+
if START in keywords and END in keywords:
|
|
636
678
|
return
|
|
637
679
|
|
|
638
|
-
|
|
680
|
+
collection_conf = getattr(self.config, "metadata_mapping", {})
|
|
639
681
|
if (
|
|
640
|
-
|
|
641
|
-
and
|
|
642
|
-
and "metadata_mapping" in self.config.products[
|
|
682
|
+
collection
|
|
683
|
+
and collection in self.config.products
|
|
684
|
+
and "metadata_mapping" in self.config.products[collection]
|
|
643
685
|
):
|
|
644
|
-
|
|
686
|
+
collection_conf = self.config.products[collection]["metadata_mapping"]
|
|
645
687
|
|
|
646
688
|
# start time given, end time missing
|
|
647
689
|
if START in keywords:
|
|
648
690
|
keywords[END] = (
|
|
649
691
|
keywords[START]
|
|
650
|
-
if END in
|
|
651
|
-
else self.
|
|
652
|
-
|
|
653
|
-
)
|
|
692
|
+
if END in collection_conf
|
|
693
|
+
# else self.get_collection_cfg_value(
|
|
694
|
+
else self.get_collection_cfg_dates(None, today().isoformat())[1]
|
|
654
695
|
)
|
|
655
696
|
return
|
|
656
697
|
|
|
657
|
-
if END in
|
|
658
|
-
mapping =
|
|
698
|
+
if END in collection_conf:
|
|
699
|
+
mapping = collection_conf[START]
|
|
659
700
|
if not isinstance(mapping, list):
|
|
660
|
-
mapping =
|
|
701
|
+
mapping = collection_conf[END]
|
|
661
702
|
if isinstance(mapping, list):
|
|
662
703
|
# if startTime is not given but other time params (e.g. year/month/(day)) are given,
|
|
663
704
|
# no default date is required
|
|
664
705
|
start, end = ecmwf_temporal_to_eodag(keywords)
|
|
665
706
|
if start is None:
|
|
666
|
-
|
|
667
|
-
|
|
707
|
+
col_start, col_end = self.get_collection_cfg_dates(
|
|
708
|
+
DEFAULT_MISSION_START_DATE, today().isoformat()
|
|
668
709
|
)
|
|
710
|
+
keywords[START] = col_start
|
|
669
711
|
keywords[END] = (
|
|
670
|
-
keywords[START]
|
|
671
|
-
if END in product_type_conf
|
|
672
|
-
else self.get_product_type_cfg_value(
|
|
673
|
-
"missionEndDate", today().isoformat()
|
|
674
|
-
)
|
|
712
|
+
keywords[START] if END in collection_conf else col_end
|
|
675
713
|
)
|
|
676
714
|
else:
|
|
677
715
|
keywords[START] = start
|
|
678
716
|
keywords[END] = end
|
|
679
717
|
|
|
680
|
-
def
|
|
681
|
-
self,
|
|
718
|
+
def _get_collection_queryables(
|
|
719
|
+
self, collection: Optional[str], alias: Optional[str], filters: dict[str, Any]
|
|
682
720
|
) -> QueryablesDict:
|
|
683
721
|
"""Override to set additional_properties to false."""
|
|
684
722
|
default_values: dict[str, Any] = deepcopy(
|
|
685
|
-
getattr(self.config, "products", {}).get(
|
|
723
|
+
getattr(self.config, "products", {}).get(collection, {})
|
|
686
724
|
)
|
|
687
725
|
default_values.pop("metadata_mapping", None)
|
|
726
|
+
default_values.pop("metadata_mapping_from_product", None)
|
|
688
727
|
|
|
689
|
-
filters["
|
|
728
|
+
filters["collection"] = collection
|
|
690
729
|
queryables = self.discover_queryables(**{**default_values, **filters}) or {}
|
|
691
730
|
|
|
692
731
|
return QueryablesDict(additional_properties=False, **queryables)
|
|
693
732
|
|
|
733
|
+
def _find_dynamic_queryables_config(
|
|
734
|
+
self, kwargs: dict[str, Any], dynamic_config: list
|
|
735
|
+
) -> dict[str, Any]:
|
|
736
|
+
"""Find the appropriate queryables configuration from dynamic configuration.
|
|
737
|
+
|
|
738
|
+
:param kwargs: Search parameters
|
|
739
|
+
:param dynamic_config: List of dynamic discover queryables configurations
|
|
740
|
+
:return: Found queryables configuration or empty dict
|
|
741
|
+
"""
|
|
742
|
+
for dc in dynamic_config:
|
|
743
|
+
for cs in dc["collection_selector"]:
|
|
744
|
+
field = cs["field"]
|
|
745
|
+
if kwargs[field].startswith(cs["prefix"]):
|
|
746
|
+
return dc["discover_queryables"]
|
|
747
|
+
return {}
|
|
748
|
+
|
|
694
749
|
def discover_queryables(
|
|
695
|
-
self,
|
|
750
|
+
self,
|
|
751
|
+
**kwargs: Any,
|
|
696
752
|
) -> Optional[dict[str, Annotated[Any, FieldInfo]]]:
|
|
697
753
|
"""Fetch queryables list from provider using its constraints file
|
|
698
754
|
|
|
699
|
-
:param kwargs: additional filters for queryables (`
|
|
755
|
+
:param kwargs: additional filters for queryables (`collection` and other search
|
|
700
756
|
arguments)
|
|
701
757
|
:returns: fetched queryable parameters dict
|
|
702
758
|
"""
|
|
703
|
-
|
|
759
|
+
collection = kwargs.pop("collection")
|
|
704
760
|
|
|
705
|
-
|
|
761
|
+
col_config = self.get_collection_def_params(collection)
|
|
706
762
|
|
|
707
|
-
default_values = deepcopy(
|
|
763
|
+
default_values = deepcopy(col_config)
|
|
708
764
|
default_values.pop("metadata_mapping", None)
|
|
765
|
+
default_values.pop("metadata_mapping_from_product", None)
|
|
766
|
+
default_values.pop("discover_queryables", None)
|
|
767
|
+
kwargs.pop("discover_queryables", None)
|
|
709
768
|
filters = {**default_values, **kwargs}
|
|
710
769
|
|
|
711
770
|
if "start" in filters:
|
|
@@ -716,29 +775,33 @@ class ECMWFSearch(PostJsonSearch):
|
|
|
716
775
|
# extract default datetime and convert geometry
|
|
717
776
|
try:
|
|
718
777
|
processed_filters = self._preprocess_search_params(
|
|
719
|
-
deepcopy(filters),
|
|
778
|
+
deepcopy(filters), collection
|
|
720
779
|
)
|
|
721
780
|
except Exception as e:
|
|
722
781
|
raise ValidationError(e.args[0]) from e
|
|
723
782
|
|
|
724
|
-
|
|
725
|
-
|
|
726
|
-
|
|
727
|
-
|
|
783
|
+
# dynamic_discover_queryables for WekeoECMWFSearch
|
|
784
|
+
queryables_config = {}
|
|
785
|
+
if dynamic_config := getattr(self.config, "dynamic_discover_queryables", []):
|
|
786
|
+
queryables_config = self._find_dynamic_queryables_config(
|
|
787
|
+
kwargs, dynamic_config
|
|
788
|
+
)
|
|
789
|
+
|
|
790
|
+
provider_dq = getattr(self.config, "discover_queryables", {}) or {}
|
|
791
|
+
product_dq = col_config.get("discover_queryables", {}) or {}
|
|
792
|
+
dq_conf = {**provider_dq, **product_dq, **queryables_config}
|
|
793
|
+
constraints_url = format_metadata(dq_conf.get("constraints_url", ""), **filters)
|
|
728
794
|
constraints: list[dict[str, Any]] = self._fetch_data(constraints_url)
|
|
729
795
|
|
|
730
|
-
form_url = format_metadata(
|
|
731
|
-
getattr(self.config, "discover_queryables", {}).get("form_url", ""),
|
|
732
|
-
**filters,
|
|
733
|
-
)
|
|
796
|
+
form_url = format_metadata(dq_conf.get("form_url", ""), **filters)
|
|
734
797
|
form: list[dict[str, Any]] = self._fetch_data(form_url)
|
|
735
798
|
|
|
736
799
|
formated_filters = self.format_as_provider_keyword(
|
|
737
|
-
|
|
800
|
+
collection, deepcopy(processed_filters)
|
|
738
801
|
)
|
|
739
802
|
# we re-apply kwargs input to consider override of year, month, day and time.
|
|
740
803
|
for k, v in {**default_values, **kwargs}.items():
|
|
741
|
-
key = k.removeprefix(ECMWF_PREFIX)
|
|
804
|
+
key = k.removeprefix(ECMWF_PREFIX).removeprefix(f"{ECMWF_PREFIX[:-1]}_")
|
|
742
805
|
|
|
743
806
|
if key not in ALLOWED_KEYWORDS | {
|
|
744
807
|
START,
|
|
@@ -781,28 +844,30 @@ class ECMWFSearch(PostJsonSearch):
|
|
|
781
844
|
else:
|
|
782
845
|
values_url = getattr(self.config, "available_values_url", "")
|
|
783
846
|
if not values_url:
|
|
784
|
-
return self.queryables_from_metadata_mapping(
|
|
847
|
+
return self.queryables_from_metadata_mapping(collection)
|
|
785
848
|
if "{" in values_url:
|
|
786
|
-
values_url = values_url
|
|
849
|
+
values_url = format_string(None, values_url, **filters)
|
|
787
850
|
data = self._fetch_data(values_url)
|
|
788
851
|
available_values = data["constraints"]
|
|
789
852
|
required_keywords = data.get("required", [])
|
|
790
853
|
|
|
791
854
|
# To check if all keywords are queryable parameters, we check if they are in the
|
|
792
|
-
# available values or the
|
|
855
|
+
# available values or the collection config (available values calculated from the
|
|
793
856
|
# constraints might not include all queryables)
|
|
794
857
|
for keyword in processed_filters:
|
|
795
858
|
if (
|
|
796
859
|
keyword
|
|
797
860
|
not in available_values.keys()
|
|
798
|
-
|
|
|
861
|
+
| col_config.keys()
|
|
799
862
|
| {
|
|
800
863
|
START,
|
|
801
864
|
END,
|
|
802
865
|
"geometry",
|
|
803
866
|
}
|
|
804
867
|
and keyword not in [f["name"] for f in form]
|
|
805
|
-
and keyword.removeprefix(ECMWF_PREFIX)
|
|
868
|
+
and keyword.removeprefix(ECMWF_PREFIX).removeprefix(
|
|
869
|
+
f"{ECMWF_PREFIX[:-1]}_"
|
|
870
|
+
)
|
|
806
871
|
not in set(list(available_values.keys()) + [f["name"] for f in form])
|
|
807
872
|
):
|
|
808
873
|
raise ValidationError(
|
|
@@ -823,10 +888,11 @@ class ECMWFSearch(PostJsonSearch):
|
|
|
823
888
|
|
|
824
889
|
# ecmwf:date is replaced by start and end.
|
|
825
890
|
# start and end filters are supported whenever combinations of "year", "month", "day" filters exist
|
|
891
|
+
queryable_prefix = f"{ECMWF_PREFIX[:-1]}_"
|
|
826
892
|
if (
|
|
827
|
-
queryables.pop(f"{
|
|
828
|
-
or f"{
|
|
829
|
-
or f"{
|
|
893
|
+
queryables.pop(f"{queryable_prefix}date", None)
|
|
894
|
+
or f"{queryable_prefix}year" in queryables
|
|
895
|
+
or f"{queryable_prefix}hyear" in queryables
|
|
830
896
|
):
|
|
831
897
|
queryables.update(
|
|
832
898
|
{
|
|
@@ -842,13 +908,7 @@ class ECMWFSearch(PostJsonSearch):
|
|
|
842
908
|
|
|
843
909
|
# area is geom in EODAG.
|
|
844
910
|
if queryables.pop("area", None):
|
|
845
|
-
queryables["geom"] =
|
|
846
|
-
Union[str, dict[str, float], BaseGeometry],
|
|
847
|
-
Field(
|
|
848
|
-
None,
|
|
849
|
-
description="Read EODAG documentation for all supported geometry format.",
|
|
850
|
-
),
|
|
851
|
-
]
|
|
911
|
+
queryables["geom"] = Queryables.get_with_default("geom", None)
|
|
852
912
|
|
|
853
913
|
return queryables
|
|
854
914
|
|
|
@@ -899,21 +959,13 @@ class ECMWFSearch(PostJsonSearch):
|
|
|
899
959
|
)
|
|
900
960
|
|
|
901
961
|
# We convert every single value to a list of string
|
|
902
|
-
filter_v =
|
|
903
|
-
filter_v = filter_v if isinstance(filter_v, list) else [filter_v]
|
|
962
|
+
filter_v = values if isinstance(values, (list, tuple)) else [values]
|
|
904
963
|
|
|
905
964
|
# We strip values of superfluous quotes (added by mapping converter to_geojson).
|
|
906
|
-
# ECMWF accept
|
|
907
|
-
# ECMWF accept
|
|
908
|
-
|
|
909
|
-
|
|
910
|
-
]
|
|
911
|
-
is_date = any(
|
|
912
|
-
any(r.match(v) is not None for r in date_regex) for v in filter_v
|
|
913
|
-
)
|
|
914
|
-
if is_date:
|
|
915
|
-
sep = re.compile(r"/to/|/")
|
|
916
|
-
filter_v = [i for v in filter_v for i in sep.split(str(v))]
|
|
965
|
+
# ECMWF accept values with /to/. We need to split it to an array
|
|
966
|
+
# ECMWF accept values in format val1/val2. We need to split it to an array
|
|
967
|
+
sep = re.compile(r"/to/|/")
|
|
968
|
+
filter_v = [i for v in filter_v for i in sep.split(str(v))]
|
|
917
969
|
|
|
918
970
|
# special handling for time 0000 converted to 0 by pre-formating with metadata_mapping
|
|
919
971
|
if keyword.split(":")[-1] == "time":
|
|
@@ -1048,16 +1100,21 @@ class ECMWFSearch(PostJsonSearch):
|
|
|
1048
1100
|
if default and prop.get("type") == "string" and isinstance(default, list):
|
|
1049
1101
|
default = ",".join(default)
|
|
1050
1102
|
|
|
1051
|
-
is_required = bool(element.get("required"))
|
|
1103
|
+
is_required = bool(element.get("required")) and bool(
|
|
1104
|
+
available_values.get(name)
|
|
1105
|
+
)
|
|
1052
1106
|
if is_required:
|
|
1053
1107
|
required_list.append(name)
|
|
1054
1108
|
|
|
1055
|
-
|
|
1109
|
+
formatted_param = ecmwf_format(name, alias=False)
|
|
1110
|
+
formatted_alias = ecmwf_format(name)
|
|
1111
|
+
queryables[formatted_param] = Annotated[
|
|
1056
1112
|
get_args(
|
|
1057
1113
|
json_field_definition_to_python(
|
|
1058
1114
|
prop,
|
|
1059
1115
|
default_value=default,
|
|
1060
1116
|
required=is_required,
|
|
1117
|
+
alias=formatted_alias,
|
|
1061
1118
|
)
|
|
1062
1119
|
)
|
|
1063
1120
|
]
|
|
@@ -1087,14 +1144,16 @@ class ECMWFSearch(PostJsonSearch):
|
|
|
1087
1144
|
for name, values in available_values.items():
|
|
1088
1145
|
# Rename keywords from form with metadata mapping.
|
|
1089
1146
|
# Needed to map constraints like "xxxx" to eodag parameter "ecmwf:xxxx"
|
|
1090
|
-
|
|
1147
|
+
formatted_param = ecmwf_format(name, alias=False)
|
|
1148
|
+
formatted_alias = ecmwf_format(name)
|
|
1091
1149
|
|
|
1092
|
-
queryables[
|
|
1150
|
+
queryables[formatted_param] = Annotated[
|
|
1093
1151
|
get_args(
|
|
1094
1152
|
json_field_definition_to_python(
|
|
1095
1153
|
{"type": "string", "title": name, "enum": values},
|
|
1096
1154
|
default_value=defaults.get(name),
|
|
1097
|
-
required=bool(
|
|
1155
|
+
required=bool(formatted_alias in required),
|
|
1156
|
+
alias=formatted_alias,
|
|
1098
1157
|
)
|
|
1099
1158
|
)
|
|
1100
1159
|
]
|
|
@@ -1102,32 +1161,32 @@ class ECMWFSearch(PostJsonSearch):
|
|
|
1102
1161
|
return queryables
|
|
1103
1162
|
|
|
1104
1163
|
def format_as_provider_keyword(
|
|
1105
|
-
self,
|
|
1164
|
+
self, collection: str, properties: dict[str, Any]
|
|
1106
1165
|
) -> dict[str, Any]:
|
|
1107
1166
|
"""Return provider equivalent keyword names from EODAG keywords.
|
|
1108
1167
|
|
|
1109
|
-
:param
|
|
1168
|
+
:param collection: collection id
|
|
1110
1169
|
:param properties: dict of properties to be formatted
|
|
1111
1170
|
:return: dict of formatted properties
|
|
1112
1171
|
"""
|
|
1113
|
-
properties["
|
|
1172
|
+
properties["collection"] = collection
|
|
1114
1173
|
|
|
1115
|
-
# provider
|
|
1116
|
-
|
|
1117
|
-
|
|
1174
|
+
# provider collection specific conf
|
|
1175
|
+
collection_def_params = self.get_collection_def_params(
|
|
1176
|
+
collection, format_variables=properties
|
|
1118
1177
|
)
|
|
1119
1178
|
|
|
1120
|
-
# Add to the query, the queryable parameters set in the provider
|
|
1179
|
+
# Add to the query, the queryable parameters set in the provider collection definition
|
|
1121
1180
|
properties.update(
|
|
1122
1181
|
{
|
|
1123
1182
|
k: v
|
|
1124
|
-
for k, v in
|
|
1183
|
+
for k, v in collection_def_params.items()
|
|
1125
1184
|
if k not in properties.keys()
|
|
1126
1185
|
and k in self.config.metadata_mapping.keys()
|
|
1127
1186
|
and isinstance(self.config.metadata_mapping[k], list)
|
|
1128
1187
|
}
|
|
1129
1188
|
)
|
|
1130
|
-
qp, _ = self.build_query_string(
|
|
1189
|
+
qp, _ = self.build_query_string(collection, properties)
|
|
1131
1190
|
|
|
1132
1191
|
return qp
|
|
1133
1192
|
|
|
@@ -1160,7 +1219,7 @@ class ECMWFSearch(PostJsonSearch):
|
|
|
1160
1219
|
:returns: list of single :class:`~eodag.api.product._product.EOProduct`
|
|
1161
1220
|
"""
|
|
1162
1221
|
|
|
1163
|
-
|
|
1222
|
+
collection = kwargs.get("collection")
|
|
1164
1223
|
|
|
1165
1224
|
result = results[0]
|
|
1166
1225
|
|
|
@@ -1178,27 +1237,27 @@ class ECMWFSearch(PostJsonSearch):
|
|
|
1178
1237
|
|
|
1179
1238
|
if result:
|
|
1180
1239
|
properties = result
|
|
1181
|
-
properties.update(result.pop("request_params", None) or {})
|
|
1240
|
+
properties.update(result.pop("eodag:request_params", None) or {})
|
|
1182
1241
|
|
|
1183
1242
|
properties = {k: v for k, v in properties.items() if not k.startswith("__")}
|
|
1184
1243
|
|
|
1185
1244
|
properties["geometry"] = properties.get("area") or DEFAULT_GEOMETRY
|
|
1186
1245
|
|
|
1187
1246
|
start, end = ecmwf_temporal_to_eodag(properties)
|
|
1188
|
-
properties["
|
|
1189
|
-
properties["
|
|
1247
|
+
properties["start_datetime"] = start
|
|
1248
|
+
properties["end_datetime"] = end
|
|
1190
1249
|
|
|
1191
1250
|
else:
|
|
1192
1251
|
# use all available query_params to parse properties
|
|
1193
1252
|
result_data: dict[str, Any] = {
|
|
1194
|
-
**results.
|
|
1253
|
+
**results.collection_def_params,
|
|
1195
1254
|
**sorted_unpaginated_qp,
|
|
1196
1255
|
**{"qs": sorted_unpaginated_qp},
|
|
1197
1256
|
}
|
|
1198
1257
|
|
|
1199
|
-
# update result with
|
|
1258
|
+
# update result with collection_def_params and search args if not None (and not auth)
|
|
1200
1259
|
kwargs.pop("auth", None)
|
|
1201
|
-
result_data.update(results.
|
|
1260
|
+
result_data.update(results.collection_def_params)
|
|
1202
1261
|
result_data = {
|
|
1203
1262
|
**result_data,
|
|
1204
1263
|
**{k: v for k, v in kwargs.items() if v is not None},
|
|
@@ -1213,17 +1272,18 @@ class ECMWFSearch(PostJsonSearch):
|
|
|
1213
1272
|
query_hash = hashlib.sha1(str(result_data).encode("UTF-8")).hexdigest()
|
|
1214
1273
|
|
|
1215
1274
|
properties["title"] = properties["id"] = (
|
|
1216
|
-
(
|
|
1275
|
+
(collection or kwargs.get("dataset", self.provider)).upper()
|
|
1217
1276
|
+ "_ORDERABLE_"
|
|
1218
1277
|
+ query_hash
|
|
1219
1278
|
)
|
|
1220
|
-
|
|
1221
|
-
|
|
1222
|
-
|
|
1279
|
+
|
|
1280
|
+
# collection alias (required by opentelemetry-instrumentation-eodag)
|
|
1281
|
+
if alias := getattr(self.config, "collection_config", {}).get("alias"):
|
|
1282
|
+
properties["eodag:alias"] = alias
|
|
1223
1283
|
|
|
1224
1284
|
qs = geojson.dumps(sorted_unpaginated_qp)
|
|
1225
1285
|
|
|
1226
|
-
# used by server mode to generate
|
|
1286
|
+
# used by server mode to generate eodag:download_link href
|
|
1227
1287
|
# TODO: to remove once the legacy server is removed
|
|
1228
1288
|
properties["_dc_qs"] = quote_plus(qs)
|
|
1229
1289
|
|
|
@@ -1272,12 +1332,12 @@ def _check_id(product: EOProduct) -> EOProduct:
|
|
|
1272
1332
|
if not on_response_mm:
|
|
1273
1333
|
return product
|
|
1274
1334
|
|
|
1275
|
-
logger.debug(f"Update product properties using given
|
|
1335
|
+
logger.debug(f"Update product properties using given eodag:order_id {product_id}")
|
|
1276
1336
|
on_response_mm_jsonpath = mtd_cfg_as_conversion_and_querypath(
|
|
1277
1337
|
on_response_mm,
|
|
1278
1338
|
)
|
|
1279
1339
|
properties_update = properties_from_json(
|
|
1280
|
-
{}, {**on_response_mm_jsonpath, **{"
|
|
1340
|
+
{}, {**on_response_mm_jsonpath, **{"eodag:order_id": (None, product_id)}}
|
|
1281
1341
|
)
|
|
1282
1342
|
product.properties.update(
|
|
1283
1343
|
{k: v for k, v in properties_update.items() if v != NOT_AVAILABLE}
|
|
@@ -1290,7 +1350,7 @@ def _check_id(product: EOProduct) -> EOProduct:
|
|
|
1290
1350
|
product.downloader._order_status(product=product, auth=auth) # type: ignore
|
|
1291
1351
|
# when a NotAvailableError is catched, it means the product is not ready and still needs to be polled
|
|
1292
1352
|
except NotAvailableError:
|
|
1293
|
-
product.properties["
|
|
1353
|
+
product.properties["order:status"] = STAGING_STATUS
|
|
1294
1354
|
except Exception as e:
|
|
1295
1355
|
if (
|
|
1296
1356
|
isinstance(e, DownloadError) or isinstance(e, ValidationError)
|
|
@@ -1302,16 +1362,16 @@ def _check_id(product: EOProduct) -> EOProduct:
|
|
|
1302
1362
|
|
|
1303
1363
|
# update product id
|
|
1304
1364
|
product.properties["id"] = product_id
|
|
1305
|
-
# update
|
|
1306
|
-
if product.
|
|
1307
|
-
product.
|
|
1365
|
+
# update collection if needed
|
|
1366
|
+
if product.collection is None:
|
|
1367
|
+
product.collection = product.properties.get("ecmwf:dataset")
|
|
1308
1368
|
# update product title
|
|
1309
1369
|
product.properties["title"] = (
|
|
1310
|
-
(product.
|
|
1370
|
+
(product.collection or product.provider).upper() + "_" + product_id
|
|
1311
1371
|
)
|
|
1312
|
-
# use NOT_AVAILABLE as fallback
|
|
1313
|
-
if product.
|
|
1314
|
-
product.
|
|
1372
|
+
# use NOT_AVAILABLE as fallback collection to avoid using guess_collection
|
|
1373
|
+
if product.collection is None:
|
|
1374
|
+
product.collection = NOT_AVAILABLE
|
|
1315
1375
|
|
|
1316
1376
|
return product
|
|
1317
1377
|
|
|
@@ -1371,7 +1431,7 @@ class MeteoblueSearch(ECMWFSearch):
|
|
|
1371
1431
|
|
|
1372
1432
|
def do_search(
|
|
1373
1433
|
self, prep: PreparedSearch = PreparedSearch(items_per_page=None), **kwargs: Any
|
|
1374
|
-
) ->
|
|
1434
|
+
) -> RawSearchResult:
|
|
1375
1435
|
"""Perform the actual search request, and return result in a single element.
|
|
1376
1436
|
|
|
1377
1437
|
:param prep: :class:`~eodag.plugins.search.PreparedSearch` object containing information for the search
|
|
@@ -1386,19 +1446,23 @@ class MeteoblueSearch(ECMWFSearch):
|
|
|
1386
1446
|
f" {self.__class__.__name__} instance"
|
|
1387
1447
|
)
|
|
1388
1448
|
response = self._request(prep)
|
|
1449
|
+
raw_search_results = RawSearchResult([response.json()])
|
|
1450
|
+
raw_search_results.search_params = kwargs
|
|
1389
1451
|
|
|
1390
|
-
|
|
1452
|
+
raw_search_results.query_params = prep.query_params
|
|
1453
|
+
raw_search_results.collection_def_params = prep.collection_def_params
|
|
1454
|
+
return raw_search_results
|
|
1391
1455
|
|
|
1392
1456
|
def build_query_string(
|
|
1393
|
-
self,
|
|
1457
|
+
self, collection: str, query_dict: dict[str, Any]
|
|
1394
1458
|
) -> tuple[dict[str, Any], str]:
|
|
1395
1459
|
"""Build The query string using the search parameters
|
|
1396
1460
|
|
|
1397
|
-
:param
|
|
1461
|
+
:param collection: collection id
|
|
1398
1462
|
:param query_dict: keyword arguments to be used in the query string
|
|
1399
1463
|
:return: formatted query params and encode query string
|
|
1400
1464
|
"""
|
|
1401
|
-
return QueryStringSearch.build_query_string(self,
|
|
1465
|
+
return QueryStringSearch.build_query_string(self, collection, query_dict)
|
|
1402
1466
|
|
|
1403
1467
|
def normalize_results(self, results, **kwargs):
|
|
1404
1468
|
"""Build :class:`~eodag.api.product._product.EOProduct` from provider result
|
|
@@ -1408,7 +1472,7 @@ class MeteoblueSearch(ECMWFSearch):
|
|
|
1408
1472
|
:returns: list of single :class:`~eodag.api.product._product.EOProduct`
|
|
1409
1473
|
"""
|
|
1410
1474
|
|
|
1411
|
-
|
|
1475
|
+
collection = kwargs.get("collection")
|
|
1412
1476
|
|
|
1413
1477
|
result = results[0]
|
|
1414
1478
|
|
|
@@ -1442,9 +1506,9 @@ class MeteoblueSearch(ECMWFSearch):
|
|
|
1442
1506
|
|
|
1443
1507
|
query_hash = hashlib.sha1(str(qs).encode("UTF-8")).hexdigest()
|
|
1444
1508
|
|
|
1445
|
-
# update result with
|
|
1509
|
+
# update result with collection_def_params and search args if not None (and not auth)
|
|
1446
1510
|
kwargs.pop("auth", None)
|
|
1447
|
-
result.update(results.
|
|
1511
|
+
result.update(results.collection_def_params)
|
|
1448
1512
|
result = dict(result, **{k: v for k, v in kwargs.items() if v is not None})
|
|
1449
1513
|
|
|
1450
1514
|
# parse properties
|
|
@@ -1454,17 +1518,16 @@ class MeteoblueSearch(ECMWFSearch):
|
|
|
1454
1518
|
discovery_config=getattr(self.config, "discover_metadata", {}),
|
|
1455
1519
|
)
|
|
1456
1520
|
|
|
1457
|
-
properties = {
|
|
1458
|
-
|
|
1459
|
-
|
|
1460
|
-
|
|
1461
|
-
}
|
|
1521
|
+
properties = {ecmwf_format(k): v for k, v in parsed_properties.items()}
|
|
1522
|
+
# collection alias (required by opentelemetry-instrumentation-eodag)
|
|
1523
|
+
if alias := getattr(self.config, "collection_config", {}).get("alias"):
|
|
1524
|
+
properties["eodag:alias"] = alias
|
|
1462
1525
|
|
|
1463
1526
|
def slugify(date_str: str) -> str:
|
|
1464
1527
|
return date_str.split("T")[0].replace("-", "")
|
|
1465
1528
|
|
|
1466
1529
|
# build product id
|
|
1467
|
-
product_id = (
|
|
1530
|
+
product_id = (collection or self.provider).upper()
|
|
1468
1531
|
|
|
1469
1532
|
start = properties.get(START, NOT_AVAILABLE)
|
|
1470
1533
|
end = properties.get(END, NOT_AVAILABLE)
|
|
@@ -1478,17 +1541,14 @@ class MeteoblueSearch(ECMWFSearch):
|
|
|
1478
1541
|
|
|
1479
1542
|
properties["id"] = properties["title"] = product_id
|
|
1480
1543
|
|
|
1481
|
-
# used by server mode to generate
|
|
1544
|
+
# used by server mode to generate eodag:download_link href
|
|
1482
1545
|
properties["_dc_qs"] = quote_plus(qs)
|
|
1483
1546
|
|
|
1484
1547
|
product = EOProduct(
|
|
1485
1548
|
provider=self.provider,
|
|
1486
|
-
|
|
1549
|
+
collection=collection,
|
|
1487
1550
|
properties=properties,
|
|
1488
1551
|
)
|
|
1489
|
-
# use product_type_config as default properties
|
|
1490
|
-
product_type_config = getattr(self.config, "product_type_config", {})
|
|
1491
|
-
product.properties = dict(product_type_config, **product.properties)
|
|
1492
1552
|
|
|
1493
1553
|
return [
|
|
1494
1554
|
product,
|
|
@@ -1534,9 +1594,9 @@ class WekeoECMWFSearch(ECMWFSearch):
|
|
|
1534
1594
|
# id is order id (only letters and numbers) -> use parent normalize results
|
|
1535
1595
|
return super().normalize_results(results, **kwargs)
|
|
1536
1596
|
|
|
1537
|
-
# formating of
|
|
1597
|
+
# formating of eodag:order_link requires access to the collection value.
|
|
1538
1598
|
results.data = [
|
|
1539
|
-
{**result, **results.
|
|
1599
|
+
{**result, **results.collection_def_params} for result in results
|
|
1540
1600
|
]
|
|
1541
1601
|
|
|
1542
1602
|
normalized = QueryStringSearch.normalize_results(self, results, **kwargs)
|
|
@@ -1561,14 +1621,16 @@ class WekeoECMWFSearch(ECMWFSearch):
|
|
|
1561
1621
|
dataset = "_".join(splitted_id[:-1])
|
|
1562
1622
|
query_hash = splitted_id[-1]
|
|
1563
1623
|
product.properties["title"] = product.properties["id"] = (
|
|
1564
|
-
(product.
|
|
1624
|
+
(product.collection or dataset or self.provider).upper()
|
|
1565
1625
|
+ "_ORDERABLE_"
|
|
1566
1626
|
+ query_hash
|
|
1567
1627
|
)
|
|
1568
1628
|
|
|
1569
1629
|
return normalized
|
|
1570
1630
|
|
|
1571
|
-
def do_search(
|
|
1631
|
+
def do_search(
|
|
1632
|
+
self, prep: PreparedSearch = PreparedSearch(items_per_page=None), **kwargs: Any
|
|
1633
|
+
) -> RawSearchResult:
|
|
1572
1634
|
"""Should perform the actual search request.
|
|
1573
1635
|
|
|
1574
1636
|
:param args: arguments to be used in the search
|
|
@@ -1578,6 +1640,16 @@ class WekeoECMWFSearch(ECMWFSearch):
|
|
|
1578
1640
|
if "id" in kwargs and "ORDERABLE" not in kwargs["id"]:
|
|
1579
1641
|
# id is order id (only letters and numbers) -> use parent normalize results.
|
|
1580
1642
|
# No real search. We fake it all, then check order status using given id
|
|
1581
|
-
|
|
1643
|
+
raw_search_results = RawSearchResult([{}])
|
|
1644
|
+
raw_search_results.search_params = kwargs
|
|
1645
|
+
raw_search_results.query_params = (
|
|
1646
|
+
prep.query_params if hasattr(prep, "query_params") else {}
|
|
1647
|
+
)
|
|
1648
|
+
raw_search_results.collection_def_params = (
|
|
1649
|
+
prep.collection_def_params
|
|
1650
|
+
if hasattr(prep, "collection_def_params")
|
|
1651
|
+
else {}
|
|
1652
|
+
)
|
|
1653
|
+
return raw_search_results
|
|
1582
1654
|
else:
|
|
1583
|
-
return QueryStringSearch.do_search(self,
|
|
1655
|
+
return QueryStringSearch.do_search(self, prep, **kwargs)
|