eodag 3.0.1__py3-none-any.whl → 3.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- eodag/api/core.py +174 -138
- eodag/api/product/_assets.py +44 -15
- eodag/api/product/_product.py +58 -47
- eodag/api/product/drivers/__init__.py +81 -4
- eodag/api/product/drivers/base.py +65 -4
- eodag/api/product/drivers/generic.py +65 -0
- eodag/api/product/drivers/sentinel1.py +97 -0
- eodag/api/product/drivers/sentinel2.py +95 -0
- eodag/api/product/metadata_mapping.py +117 -90
- eodag/api/search_result.py +13 -23
- eodag/cli.py +26 -5
- eodag/config.py +86 -92
- eodag/plugins/apis/base.py +1 -1
- eodag/plugins/apis/ecmwf.py +42 -22
- eodag/plugins/apis/usgs.py +17 -16
- eodag/plugins/authentication/aws_auth.py +16 -13
- eodag/plugins/authentication/base.py +5 -3
- eodag/plugins/authentication/header.py +3 -3
- eodag/plugins/authentication/keycloak.py +4 -4
- eodag/plugins/authentication/oauth.py +7 -3
- eodag/plugins/authentication/openid_connect.py +22 -16
- eodag/plugins/authentication/sas_auth.py +4 -4
- eodag/plugins/authentication/token.py +41 -10
- eodag/plugins/authentication/token_exchange.py +1 -1
- eodag/plugins/base.py +4 -4
- eodag/plugins/crunch/base.py +4 -4
- eodag/plugins/crunch/filter_date.py +4 -4
- eodag/plugins/crunch/filter_latest_intersect.py +6 -6
- eodag/plugins/crunch/filter_latest_tpl_name.py +7 -7
- eodag/plugins/crunch/filter_overlap.py +4 -4
- eodag/plugins/crunch/filter_property.py +6 -7
- eodag/plugins/download/aws.py +146 -87
- eodag/plugins/download/base.py +38 -56
- eodag/plugins/download/creodias_s3.py +29 -0
- eodag/plugins/download/http.py +173 -183
- eodag/plugins/download/s3rest.py +10 -11
- eodag/plugins/manager.py +10 -20
- eodag/plugins/search/__init__.py +6 -5
- eodag/plugins/search/base.py +90 -46
- eodag/plugins/search/build_search_result.py +1048 -361
- eodag/plugins/search/cop_marine.py +22 -12
- eodag/plugins/search/creodias_s3.py +9 -73
- eodag/plugins/search/csw.py +11 -11
- eodag/plugins/search/data_request_search.py +19 -18
- eodag/plugins/search/qssearch.py +99 -258
- eodag/plugins/search/stac_list_assets.py +85 -0
- eodag/plugins/search/static_stac_search.py +4 -4
- eodag/resources/ext_product_types.json +1 -1
- eodag/resources/product_types.yml +1134 -325
- eodag/resources/providers.yml +906 -2006
- eodag/resources/stac_api.yml +2 -2
- eodag/resources/user_conf_template.yml +10 -9
- eodag/rest/cache.py +2 -2
- eodag/rest/config.py +3 -3
- eodag/rest/core.py +112 -82
- eodag/rest/errors.py +5 -5
- eodag/rest/server.py +33 -14
- eodag/rest/stac.py +41 -38
- eodag/rest/types/collections_search.py +3 -3
- eodag/rest/types/eodag_search.py +29 -23
- eodag/rest/types/queryables.py +42 -31
- eodag/rest/types/stac_search.py +15 -25
- eodag/rest/utils/__init__.py +14 -21
- eodag/rest/utils/cql_evaluate.py +6 -6
- eodag/rest/utils/rfc3339.py +2 -2
- eodag/types/__init__.py +141 -32
- eodag/types/bbox.py +2 -2
- eodag/types/download_args.py +3 -3
- eodag/types/queryables.py +183 -72
- eodag/types/search_args.py +4 -4
- eodag/types/whoosh.py +127 -3
- eodag/utils/__init__.py +153 -51
- eodag/utils/exceptions.py +28 -21
- eodag/utils/import_system.py +2 -2
- eodag/utils/repr.py +65 -6
- eodag/utils/requests.py +13 -13
- eodag/utils/rest.py +2 -2
- eodag/utils/s3.py +231 -0
- eodag/utils/stac_reader.py +10 -10
- {eodag-3.0.1.dist-info → eodag-3.1.0.dist-info}/METADATA +77 -76
- eodag-3.1.0.dist-info/RECORD +113 -0
- {eodag-3.0.1.dist-info → eodag-3.1.0.dist-info}/WHEEL +1 -1
- {eodag-3.0.1.dist-info → eodag-3.1.0.dist-info}/entry_points.txt +4 -2
- eodag/utils/constraints.py +0 -244
- eodag-3.0.1.dist-info/RECORD +0 -109
- {eodag-3.0.1.dist-info → eodag-3.1.0.dist-info}/LICENSE +0 -0
- {eodag-3.0.1.dist-info → eodag-3.1.0.dist-info}/top_level.txt +0 -0
eodag/api/core.py
CHANGED
|
@@ -23,28 +23,16 @@ import os
|
|
|
23
23
|
import re
|
|
24
24
|
import shutil
|
|
25
25
|
import tempfile
|
|
26
|
+
from importlib.metadata import version
|
|
27
|
+
from importlib.resources import files as res_files
|
|
26
28
|
from operator import itemgetter
|
|
27
|
-
from typing import
|
|
28
|
-
TYPE_CHECKING,
|
|
29
|
-
Annotated,
|
|
30
|
-
Any,
|
|
31
|
-
Dict,
|
|
32
|
-
Iterator,
|
|
33
|
-
List,
|
|
34
|
-
Optional,
|
|
35
|
-
Set,
|
|
36
|
-
Tuple,
|
|
37
|
-
Union,
|
|
38
|
-
)
|
|
29
|
+
from typing import TYPE_CHECKING, Any, Iterator, Optional, Union
|
|
39
30
|
|
|
40
31
|
import geojson
|
|
41
|
-
import pkg_resources
|
|
42
32
|
import yaml.parser
|
|
43
|
-
from pkg_resources import resource_filename
|
|
44
|
-
from pydantic.fields import FieldInfo
|
|
45
33
|
from whoosh import analysis, fields
|
|
46
34
|
from whoosh.fields import Schema
|
|
47
|
-
from whoosh.index import
|
|
35
|
+
from whoosh.index import exists_in, open_dir
|
|
48
36
|
from whoosh.qparser import QueryParser
|
|
49
37
|
|
|
50
38
|
from eodag.api.product.metadata_mapping import (
|
|
@@ -69,11 +57,11 @@ from eodag.config import (
|
|
|
69
57
|
)
|
|
70
58
|
from eodag.plugins.manager import PluginManager
|
|
71
59
|
from eodag.plugins.search import PreparedSearch
|
|
72
|
-
from eodag.plugins.search.build_search_result import
|
|
60
|
+
from eodag.plugins.search.build_search_result import MeteoblueSearch
|
|
73
61
|
from eodag.plugins.search.qssearch import PostJsonSearch
|
|
74
62
|
from eodag.types import model_fields_to_annotated
|
|
75
|
-
from eodag.types.queryables import CommonQueryables
|
|
76
|
-
from eodag.types.whoosh import EODAGQueryParser
|
|
63
|
+
from eodag.types.queryables import CommonQueryables, QueryablesDict
|
|
64
|
+
from eodag.types.whoosh import EODAGQueryParser, create_in
|
|
77
65
|
from eodag.utils import (
|
|
78
66
|
DEFAULT_DOWNLOAD_TIMEOUT,
|
|
79
67
|
DEFAULT_DOWNLOAD_WAIT,
|
|
@@ -84,7 +72,6 @@ from eodag.utils import (
|
|
|
84
72
|
HTTP_REQ_TIMEOUT,
|
|
85
73
|
MockResponse,
|
|
86
74
|
_deprecated,
|
|
87
|
-
copy_deepcopy,
|
|
88
75
|
get_geometry_from_various,
|
|
89
76
|
makedirs,
|
|
90
77
|
obj_md5sum,
|
|
@@ -93,6 +80,7 @@ from eodag.utils import (
|
|
|
93
80
|
uri_to_path,
|
|
94
81
|
)
|
|
95
82
|
from eodag.utils.exceptions import (
|
|
83
|
+
AuthenticationError,
|
|
96
84
|
EodagError,
|
|
97
85
|
NoMatchingProductType,
|
|
98
86
|
PluginImplementationError,
|
|
@@ -131,8 +119,8 @@ class EODataAccessGateway:
|
|
|
131
119
|
user_conf_file_path: Optional[str] = None,
|
|
132
120
|
locations_conf_path: Optional[str] = None,
|
|
133
121
|
) -> None:
|
|
134
|
-
product_types_config_path =
|
|
135
|
-
"eodag"
|
|
122
|
+
product_types_config_path = os.getenv("EODAG_PRODUCT_TYPES_CFG_FILE") or str(
|
|
123
|
+
res_files("eodag") / "resources" / "product_types.yml"
|
|
136
124
|
)
|
|
137
125
|
self.product_types_config = SimpleYamlProxyConfig(product_types_config_path)
|
|
138
126
|
self.product_types_config_md5 = obj_md5sum(self.product_types_config.source)
|
|
@@ -173,8 +161,8 @@ class EODataAccessGateway:
|
|
|
173
161
|
user_conf_file_path = standard_configuration_path
|
|
174
162
|
if not os.path.isfile(standard_configuration_path):
|
|
175
163
|
shutil.copy(
|
|
176
|
-
|
|
177
|
-
"eodag"
|
|
164
|
+
str(
|
|
165
|
+
res_files("eodag") / "resources" / "user_conf_template.yml"
|
|
178
166
|
),
|
|
179
167
|
standard_configuration_path,
|
|
180
168
|
)
|
|
@@ -197,7 +185,7 @@ class EODataAccessGateway:
|
|
|
197
185
|
self._plugins_manager.rebuild(self.providers_config)
|
|
198
186
|
|
|
199
187
|
# store pruned providers configs
|
|
200
|
-
self._pruned_providers_config:
|
|
188
|
+
self._pruned_providers_config: dict[str, Any] = {}
|
|
201
189
|
# filter out providers needing auth that have no credentials set
|
|
202
190
|
self._prune_providers_list()
|
|
203
191
|
|
|
@@ -215,13 +203,13 @@ class EODataAccessGateway:
|
|
|
215
203
|
locations_conf_path = os.path.join(self.conf_dir, "locations.yml")
|
|
216
204
|
if not os.path.isfile(locations_conf_path):
|
|
217
205
|
# copy locations conf file and replace path example
|
|
218
|
-
locations_conf_template =
|
|
219
|
-
"eodag"
|
|
220
|
-
os.path.join("resources", "locations_conf_template.yml"),
|
|
206
|
+
locations_conf_template = str(
|
|
207
|
+
res_files("eodag") / "resources" / "locations_conf_template.yml"
|
|
221
208
|
)
|
|
222
|
-
with
|
|
223
|
-
|
|
224
|
-
|
|
209
|
+
with (
|
|
210
|
+
open(locations_conf_template) as infile,
|
|
211
|
+
open(locations_conf_path, "w") as outfile,
|
|
212
|
+
):
|
|
225
213
|
# The template contains paths in the form of:
|
|
226
214
|
# /path/to/locations/file.shp
|
|
227
215
|
path_template = "/path/to/locations/"
|
|
@@ -233,14 +221,14 @@ class EODataAccessGateway:
|
|
|
233
221
|
outfile.write(line)
|
|
234
222
|
# copy sample shapefile dir
|
|
235
223
|
shutil.copytree(
|
|
236
|
-
|
|
224
|
+
str(res_files("eodag") / "resources" / "shp"),
|
|
237
225
|
os.path.join(self.conf_dir, "shp"),
|
|
238
226
|
)
|
|
239
227
|
self.set_locations_conf(locations_conf_path)
|
|
240
228
|
|
|
241
229
|
def get_version(self) -> str:
|
|
242
230
|
"""Get eodag package version"""
|
|
243
|
-
return
|
|
231
|
+
return version("eodag")
|
|
244
232
|
|
|
245
233
|
def build_index(self) -> None:
|
|
246
234
|
"""Build a `Whoosh <https://whoosh.readthedocs.io/en/latest/index.html>`_
|
|
@@ -317,13 +305,18 @@ class EODataAccessGateway:
|
|
|
317
305
|
product_type, **{"md5": self.product_types_config_md5}
|
|
318
306
|
)
|
|
319
307
|
# add to index
|
|
320
|
-
|
|
321
|
-
|
|
322
|
-
|
|
323
|
-
|
|
324
|
-
|
|
325
|
-
|
|
326
|
-
|
|
308
|
+
try:
|
|
309
|
+
ix_writer.add_document(
|
|
310
|
+
**{
|
|
311
|
+
k: v
|
|
312
|
+
for k, v in versioned_product_type.items()
|
|
313
|
+
if k in product_types_schema.names()
|
|
314
|
+
}
|
|
315
|
+
)
|
|
316
|
+
except TypeError as e:
|
|
317
|
+
logger.error(
|
|
318
|
+
f"Cannot write product type {product_type['ID']} into index. e={e} product_type={product_type}"
|
|
319
|
+
)
|
|
327
320
|
ix_writer.commit()
|
|
328
321
|
|
|
329
322
|
def set_preferred_provider(self, provider: str) -> None:
|
|
@@ -341,7 +334,7 @@ class EODataAccessGateway:
|
|
|
341
334
|
new_priority = max_priority + 1
|
|
342
335
|
self._plugins_manager.set_priority(provider, new_priority)
|
|
343
336
|
|
|
344
|
-
def get_preferred_provider(self) ->
|
|
337
|
+
def get_preferred_provider(self) -> tuple[str, int]:
|
|
345
338
|
"""Get the provider currently set as the preferred one for searching
|
|
346
339
|
products, along with its priority.
|
|
347
340
|
|
|
@@ -357,7 +350,7 @@ class EODataAccessGateway:
|
|
|
357
350
|
def update_providers_config(
|
|
358
351
|
self,
|
|
359
352
|
yaml_conf: Optional[str] = None,
|
|
360
|
-
dict_conf: Optional[
|
|
353
|
+
dict_conf: Optional[dict[str, Any]] = None,
|
|
361
354
|
) -> None:
|
|
362
355
|
"""Update providers configuration with given input.
|
|
363
356
|
Can be used to add a provider to existing configuration or update
|
|
@@ -403,12 +396,12 @@ class EODataAccessGateway:
|
|
|
403
396
|
name: str,
|
|
404
397
|
url: Optional[str] = None,
|
|
405
398
|
priority: Optional[int] = None,
|
|
406
|
-
search:
|
|
407
|
-
products:
|
|
399
|
+
search: dict[str, Any] = {"type": "StacSearch"},
|
|
400
|
+
products: dict[str, Any] = {
|
|
408
401
|
GENERIC_PRODUCT_TYPE: {"productType": "{productType}"}
|
|
409
402
|
},
|
|
410
|
-
download:
|
|
411
|
-
**kwargs:
|
|
403
|
+
download: dict[str, Any] = {"type": "HTTPDownload", "auth_error_code": 401},
|
|
404
|
+
**kwargs: dict[str, Any],
|
|
412
405
|
):
|
|
413
406
|
"""Adds a new provider.
|
|
414
407
|
|
|
@@ -427,7 +420,7 @@ class EODataAccessGateway:
|
|
|
427
420
|
:param download: Download :class:`~eodag.config.PluginConfig` mapping
|
|
428
421
|
:param kwargs: Additional :class:`~eodag.config.ProviderConfig` mapping
|
|
429
422
|
"""
|
|
430
|
-
conf_dict:
|
|
423
|
+
conf_dict: dict[str, Any] = {
|
|
431
424
|
name: {
|
|
432
425
|
"url": url,
|
|
433
426
|
"search": {"type": "StacSearch", **search},
|
|
@@ -571,7 +564,7 @@ class EODataAccessGateway:
|
|
|
571
564
|
main_locations_config = locations_config[main_key]
|
|
572
565
|
|
|
573
566
|
logger.info("Locations configuration loaded from %s" % locations_conf_path)
|
|
574
|
-
self.locations_config:
|
|
567
|
+
self.locations_config: list[dict[str, Any]] = main_locations_config
|
|
575
568
|
else:
|
|
576
569
|
logger.info(
|
|
577
570
|
"Could not load locations configuration from %s" % locations_conf_path
|
|
@@ -580,7 +573,7 @@ class EODataAccessGateway:
|
|
|
580
573
|
|
|
581
574
|
def list_product_types(
|
|
582
575
|
self, provider: Optional[str] = None, fetch_providers: bool = True
|
|
583
|
-
) ->
|
|
576
|
+
) -> list[dict[str, Any]]:
|
|
584
577
|
"""Lists supported product types.
|
|
585
578
|
|
|
586
579
|
:param provider: (optional) The name of a provider that must support the product
|
|
@@ -594,7 +587,7 @@ class EODataAccessGateway:
|
|
|
594
587
|
# First, update product types list if possible
|
|
595
588
|
self.fetch_product_types_list(provider=provider)
|
|
596
589
|
|
|
597
|
-
product_types:
|
|
590
|
+
product_types: list[dict[str, Any]] = []
|
|
598
591
|
|
|
599
592
|
providers_configs = (
|
|
600
593
|
list(self.providers_config.values())
|
|
@@ -650,7 +643,7 @@ class EODataAccessGateway:
|
|
|
650
643
|
providers_to_fetch = [provider]
|
|
651
644
|
|
|
652
645
|
# providers discovery confs that are fetchable
|
|
653
|
-
providers_discovery_configs_fetchable:
|
|
646
|
+
providers_discovery_configs_fetchable: dict[str, Any] = {}
|
|
654
647
|
# check if any provider has not already been fetched for product types
|
|
655
648
|
already_fetched = True
|
|
656
649
|
for provider_to_fetch in providers_to_fetch:
|
|
@@ -773,7 +766,7 @@ class EODataAccessGateway:
|
|
|
773
766
|
|
|
774
767
|
def discover_product_types(
|
|
775
768
|
self, provider: Optional[str] = None
|
|
776
|
-
) -> Optional[
|
|
769
|
+
) -> Optional[dict[str, Any]]:
|
|
777
770
|
"""Fetch providers for product types
|
|
778
771
|
|
|
779
772
|
:param provider: The name of a provider or provider-group to fetch. Defaults to
|
|
@@ -793,7 +786,7 @@ class EODataAccessGateway:
|
|
|
793
786
|
raise UnsupportedProvider(
|
|
794
787
|
f"The requested provider is not (yet) supported: {provider}"
|
|
795
788
|
)
|
|
796
|
-
ext_product_types_conf:
|
|
789
|
+
ext_product_types_conf: dict[str, Any] = {}
|
|
797
790
|
providers_to_fetch = [
|
|
798
791
|
p
|
|
799
792
|
for p in (
|
|
@@ -806,7 +799,7 @@ class EODataAccessGateway:
|
|
|
806
799
|
else self.available_providers()
|
|
807
800
|
)
|
|
808
801
|
]
|
|
809
|
-
kwargs:
|
|
802
|
+
kwargs: dict[str, Any] = {}
|
|
810
803
|
for provider in providers_to_fetch:
|
|
811
804
|
if hasattr(self.providers_config[provider], "search"):
|
|
812
805
|
search_plugin_config = self.providers_config[provider].search
|
|
@@ -847,7 +840,7 @@ class EODataAccessGateway:
|
|
|
847
840
|
return sort_dict(ext_product_types_conf)
|
|
848
841
|
|
|
849
842
|
def update_product_types_list(
|
|
850
|
-
self, ext_product_types_conf:
|
|
843
|
+
self, ext_product_types_conf: dict[str, Optional[dict[str, dict[str, Any]]]]
|
|
851
844
|
) -> None:
|
|
852
845
|
"""Update eodag product types list
|
|
853
846
|
|
|
@@ -875,7 +868,7 @@ class EODataAccessGateway:
|
|
|
875
868
|
provider,
|
|
876
869
|
)
|
|
877
870
|
continue
|
|
878
|
-
new_product_types:
|
|
871
|
+
new_product_types: list[str] = []
|
|
879
872
|
for (
|
|
880
873
|
new_product_type,
|
|
881
874
|
new_product_type_conf,
|
|
@@ -938,7 +931,7 @@ class EODataAccessGateway:
|
|
|
938
931
|
|
|
939
932
|
def available_providers(
|
|
940
933
|
self, product_type: Optional[str] = None, by_group: bool = False
|
|
941
|
-
) ->
|
|
934
|
+
) -> list[str]:
|
|
942
935
|
"""Gives the sorted list of the available providers or groups
|
|
943
936
|
|
|
944
937
|
The providers or groups are sorted first by their priority level in descending order,
|
|
@@ -965,7 +958,7 @@ class EODataAccessGateway:
|
|
|
965
958
|
|
|
966
959
|
# If by_group is True, keep only the highest priority for each group
|
|
967
960
|
if by_group:
|
|
968
|
-
group_priority:
|
|
961
|
+
group_priority: dict[str, int] = {}
|
|
969
962
|
for name, priority in providers:
|
|
970
963
|
if name not in group_priority or priority > group_priority[name]:
|
|
971
964
|
group_priority[name] = priority
|
|
@@ -1032,7 +1025,7 @@ class EODataAccessGateway:
|
|
|
1032
1025
|
missionStartDate: Optional[str] = None,
|
|
1033
1026
|
missionEndDate: Optional[str] = None,
|
|
1034
1027
|
**kwargs: Any,
|
|
1035
|
-
) ->
|
|
1028
|
+
) -> list[str]:
|
|
1036
1029
|
"""
|
|
1037
1030
|
Find EODAG product type IDs that best match a set of search parameters.
|
|
1038
1031
|
|
|
@@ -1090,7 +1083,7 @@ class EODataAccessGateway:
|
|
|
1090
1083
|
query = p.parse(text)
|
|
1091
1084
|
results = searcher.search(query, limit=None)
|
|
1092
1085
|
|
|
1093
|
-
guesses:
|
|
1086
|
+
guesses: list[dict[str, str]] = [dict(r) for r in results or []]
|
|
1094
1087
|
|
|
1095
1088
|
# datetime filtering
|
|
1096
1089
|
if missionStartDate or missionEndDate:
|
|
@@ -1131,8 +1124,8 @@ class EODataAccessGateway:
|
|
|
1131
1124
|
raise_errors: bool = False,
|
|
1132
1125
|
start: Optional[str] = None,
|
|
1133
1126
|
end: Optional[str] = None,
|
|
1134
|
-
geom: Optional[Union[str,
|
|
1135
|
-
locations: Optional[
|
|
1127
|
+
geom: Optional[Union[str, dict[str, float], BaseGeometry]] = None,
|
|
1128
|
+
locations: Optional[dict[str, str]] = None,
|
|
1136
1129
|
provider: Optional[str] = None,
|
|
1137
1130
|
count: bool = False,
|
|
1138
1131
|
**kwargs: Any,
|
|
@@ -1211,7 +1204,7 @@ class EODataAccessGateway:
|
|
|
1211
1204
|
items_per_page=items_per_page,
|
|
1212
1205
|
)
|
|
1213
1206
|
|
|
1214
|
-
errors:
|
|
1207
|
+
errors: list[tuple[str, Exception]] = []
|
|
1215
1208
|
# Loop over available providers and return the first non-empty results
|
|
1216
1209
|
for i, search_plugin in enumerate(search_plugins):
|
|
1217
1210
|
search_plugin.clear()
|
|
@@ -1240,8 +1233,8 @@ class EODataAccessGateway:
|
|
|
1240
1233
|
items_per_page: int = DEFAULT_ITEMS_PER_PAGE,
|
|
1241
1234
|
start: Optional[str] = None,
|
|
1242
1235
|
end: Optional[str] = None,
|
|
1243
|
-
geom: Optional[Union[str,
|
|
1244
|
-
locations: Optional[
|
|
1236
|
+
geom: Optional[Union[str, dict[str, float], BaseGeometry]] = None,
|
|
1237
|
+
locations: Optional[dict[str, str]] = None,
|
|
1245
1238
|
**kwargs: Any,
|
|
1246
1239
|
) -> Iterator[SearchResult]:
|
|
1247
1240
|
"""Iterate over the pages of a products search.
|
|
@@ -1417,8 +1410,8 @@ class EODataAccessGateway:
|
|
|
1417
1410
|
items_per_page: Optional[int] = None,
|
|
1418
1411
|
start: Optional[str] = None,
|
|
1419
1412
|
end: Optional[str] = None,
|
|
1420
|
-
geom: Optional[Union[str,
|
|
1421
|
-
locations: Optional[
|
|
1413
|
+
geom: Optional[Union[str, dict[str, float], BaseGeometry]] = None,
|
|
1414
|
+
locations: Optional[dict[str, str]] = None,
|
|
1422
1415
|
**kwargs: Any,
|
|
1423
1416
|
) -> SearchResult:
|
|
1424
1417
|
"""Search and return all the products matching the search criteria.
|
|
@@ -1639,7 +1632,7 @@ class EODataAccessGateway:
|
|
|
1639
1632
|
if not getattr(plugin.config, "discover_product_types", {}).get("fetch_url"):
|
|
1640
1633
|
return None
|
|
1641
1634
|
|
|
1642
|
-
kwargs:
|
|
1635
|
+
kwargs: dict[str, Any] = {"productType": product_type}
|
|
1643
1636
|
|
|
1644
1637
|
# append auth if needed
|
|
1645
1638
|
if getattr(plugin.config, "need_auth", False):
|
|
@@ -1657,11 +1650,11 @@ class EODataAccessGateway:
|
|
|
1657
1650
|
self,
|
|
1658
1651
|
start: Optional[str] = None,
|
|
1659
1652
|
end: Optional[str] = None,
|
|
1660
|
-
geom: Optional[Union[str,
|
|
1661
|
-
locations: Optional[
|
|
1653
|
+
geom: Optional[Union[str, dict[str, float], BaseGeometry]] = None,
|
|
1654
|
+
locations: Optional[dict[str, str]] = None,
|
|
1662
1655
|
provider: Optional[str] = None,
|
|
1663
1656
|
**kwargs: Any,
|
|
1664
|
-
) ->
|
|
1657
|
+
) -> tuple[list[Union[Search, Api]], dict[str, Any]]:
|
|
1665
1658
|
"""Internal method to prepare the search kwargs and get the search plugins.
|
|
1666
1659
|
|
|
1667
1660
|
Product query:
|
|
@@ -1769,16 +1762,16 @@ class EODataAccessGateway:
|
|
|
1769
1762
|
|
|
1770
1763
|
preferred_provider = self.get_preferred_provider()[0]
|
|
1771
1764
|
|
|
1772
|
-
search_plugins:
|
|
1765
|
+
search_plugins: list[Union[Search, Api]] = []
|
|
1773
1766
|
for plugin in self._plugins_manager.get_search_plugins(
|
|
1774
1767
|
product_type=product_type, provider=provider
|
|
1775
1768
|
):
|
|
1776
|
-
# exclude
|
|
1769
|
+
# exclude MeteoblueSearch plugins from search fallback for unknown product_type
|
|
1777
1770
|
if (
|
|
1778
1771
|
provider != plugin.provider
|
|
1779
1772
|
and preferred_provider != plugin.provider
|
|
1780
1773
|
and product_type not in self.product_types_config
|
|
1781
|
-
and isinstance(plugin,
|
|
1774
|
+
and isinstance(plugin, MeteoblueSearch)
|
|
1782
1775
|
):
|
|
1783
1776
|
continue
|
|
1784
1777
|
search_plugins.append(plugin)
|
|
@@ -1801,27 +1794,7 @@ class EODataAccessGateway:
|
|
|
1801
1794
|
# Add product_types_config to plugin config. This dict contains product
|
|
1802
1795
|
# type metadata that will also be stored in each product's properties.
|
|
1803
1796
|
for search_plugin in search_plugins:
|
|
1804
|
-
|
|
1805
|
-
search_plugin.config.product_type_config = dict(
|
|
1806
|
-
[
|
|
1807
|
-
p
|
|
1808
|
-
for p in self.list_product_types(
|
|
1809
|
-
search_plugin.provider, fetch_providers=False
|
|
1810
|
-
)
|
|
1811
|
-
if p["_id"] == product_type
|
|
1812
|
-
][0],
|
|
1813
|
-
**{"productType": product_type},
|
|
1814
|
-
)
|
|
1815
|
-
# If the product isn't in the catalog, it's a generic product type.
|
|
1816
|
-
except IndexError:
|
|
1817
|
-
# Construct the GENERIC_PRODUCT_TYPE metadata
|
|
1818
|
-
search_plugin.config.product_type_config = dict(
|
|
1819
|
-
ID=GENERIC_PRODUCT_TYPE,
|
|
1820
|
-
**self.product_types_config[GENERIC_PRODUCT_TYPE],
|
|
1821
|
-
productType=product_type,
|
|
1822
|
-
)
|
|
1823
|
-
# Remove the ID since this is equal to productType.
|
|
1824
|
-
search_plugin.config.product_type_config.pop("ID", None)
|
|
1797
|
+
self._attach_product_type_config(search_plugin, product_type)
|
|
1825
1798
|
|
|
1826
1799
|
return search_plugins, kwargs
|
|
1827
1800
|
|
|
@@ -1859,10 +1832,10 @@ class EODataAccessGateway:
|
|
|
1859
1832
|
max_items_per_page,
|
|
1860
1833
|
)
|
|
1861
1834
|
|
|
1862
|
-
results:
|
|
1835
|
+
results: list[EOProduct] = []
|
|
1863
1836
|
total_results: Optional[int] = 0 if count else None
|
|
1864
1837
|
|
|
1865
|
-
errors:
|
|
1838
|
+
errors: list[tuple[str, Exception]] = []
|
|
1866
1839
|
|
|
1867
1840
|
try:
|
|
1868
1841
|
prep = PreparedSearch(count=count)
|
|
@@ -2010,7 +1983,7 @@ class EODataAccessGateway:
|
|
|
2010
1983
|
return results
|
|
2011
1984
|
|
|
2012
1985
|
@staticmethod
|
|
2013
|
-
def group_by_extent(searches:
|
|
1986
|
+
def group_by_extent(searches: list[SearchResult]) -> list[SearchResult]:
|
|
2014
1987
|
"""Combines multiple SearchResults and return a list of SearchResults grouped
|
|
2015
1988
|
by extent (i.e. bounding box).
|
|
2016
1989
|
|
|
@@ -2019,7 +1992,7 @@ class EODataAccessGateway:
|
|
|
2019
1992
|
"""
|
|
2020
1993
|
# Dict with extents as keys, each extent being defined by a str
|
|
2021
1994
|
# "{minx}{miny}{maxx}{maxy}" (each float rounded to 2 dec).
|
|
2022
|
-
products_grouped_by_extent:
|
|
1995
|
+
products_grouped_by_extent: dict[str, Any] = {}
|
|
2023
1996
|
|
|
2024
1997
|
for search in searches:
|
|
2025
1998
|
for product in search:
|
|
@@ -2038,10 +2011,10 @@ class EODataAccessGateway:
|
|
|
2038
2011
|
search_result: SearchResult,
|
|
2039
2012
|
downloaded_callback: Optional[DownloadedCallback] = None,
|
|
2040
2013
|
progress_callback: Optional[ProgressCallback] = None,
|
|
2041
|
-
wait:
|
|
2042
|
-
timeout:
|
|
2014
|
+
wait: float = DEFAULT_DOWNLOAD_WAIT,
|
|
2015
|
+
timeout: float = DEFAULT_DOWNLOAD_TIMEOUT,
|
|
2043
2016
|
**kwargs: Unpack[DownloadConf],
|
|
2044
|
-
) ->
|
|
2017
|
+
) -> list[str]:
|
|
2045
2018
|
"""Download all products resulting from a search.
|
|
2046
2019
|
|
|
2047
2020
|
:param search_result: A collection of EO products resulting from a search
|
|
@@ -2201,8 +2174,8 @@ class EODataAccessGateway:
|
|
|
2201
2174
|
self,
|
|
2202
2175
|
product: EOProduct,
|
|
2203
2176
|
progress_callback: Optional[ProgressCallback] = None,
|
|
2204
|
-
wait:
|
|
2205
|
-
timeout:
|
|
2177
|
+
wait: float = DEFAULT_DOWNLOAD_WAIT,
|
|
2178
|
+
timeout: float = DEFAULT_DOWNLOAD_TIMEOUT,
|
|
2206
2179
|
**kwargs: Unpack[DownloadConf],
|
|
2207
2180
|
) -> str:
|
|
2208
2181
|
"""Download a single product.
|
|
@@ -2280,71 +2253,107 @@ class EODataAccessGateway:
|
|
|
2280
2253
|
return self._plugins_manager.get_crunch_plugin(name, **plugin_conf)
|
|
2281
2254
|
|
|
2282
2255
|
def list_queryables(
|
|
2283
|
-
self,
|
|
2284
|
-
|
|
2256
|
+
self,
|
|
2257
|
+
provider: Optional[str] = None,
|
|
2258
|
+
fetch_providers: bool = True,
|
|
2259
|
+
**kwargs: Any,
|
|
2260
|
+
) -> QueryablesDict:
|
|
2285
2261
|
"""Fetch the queryable properties for a given product type and/or provider.
|
|
2286
2262
|
|
|
2287
2263
|
:param provider: (optional) The provider.
|
|
2264
|
+
:param fetch_providers: If new product types should be fetched from the providers; default: True
|
|
2288
2265
|
:param kwargs: additional filters for queryables (`productType` or other search
|
|
2289
2266
|
arguments)
|
|
2290
2267
|
|
|
2291
2268
|
:raises UnsupportedProductType: If the specified product type is not available for the
|
|
2292
2269
|
provider.
|
|
2293
2270
|
|
|
2294
|
-
:returns: A
|
|
2295
|
-
parameters to their annotated type
|
|
2271
|
+
:returns: A :class:`~eodag.api.product.queryables.QuerybalesDict` containing the EODAG queryable
|
|
2272
|
+
properties, associating parameters to their annotated type, and a additional_properties attribute
|
|
2296
2273
|
"""
|
|
2297
|
-
|
|
2274
|
+
# only fetch providers if product type is not found
|
|
2275
|
+
available_product_types: list[str] = [
|
|
2298
2276
|
pt["ID"]
|
|
2299
2277
|
for pt in self.list_product_types(provider=provider, fetch_providers=False)
|
|
2300
2278
|
]
|
|
2301
|
-
product_type = kwargs.get("productType")
|
|
2279
|
+
product_type: Optional[str] = kwargs.get("productType")
|
|
2280
|
+
pt_alias: Optional[str] = product_type
|
|
2302
2281
|
|
|
2303
2282
|
if product_type:
|
|
2283
|
+
if product_type not in available_product_types:
|
|
2284
|
+
if fetch_providers:
|
|
2285
|
+
# fetch providers and try again
|
|
2286
|
+
available_product_types = [
|
|
2287
|
+
pt["ID"]
|
|
2288
|
+
for pt in self.list_product_types(
|
|
2289
|
+
provider=provider, fetch_providers=True
|
|
2290
|
+
)
|
|
2291
|
+
]
|
|
2292
|
+
raise UnsupportedProductType(f"{product_type} is not available.")
|
|
2304
2293
|
try:
|
|
2305
2294
|
kwargs["productType"] = product_type = self.get_product_type_from_alias(
|
|
2306
2295
|
product_type
|
|
2307
2296
|
)
|
|
2308
2297
|
except NoMatchingProductType as e:
|
|
2309
|
-
raise UnsupportedProductType(f"{product_type} is not available") from e
|
|
2310
|
-
|
|
2311
|
-
if product_type and product_type not in available_product_types:
|
|
2312
|
-
self.fetch_product_types_list()
|
|
2298
|
+
raise UnsupportedProductType(f"{product_type} is not available.") from e
|
|
2313
2299
|
|
|
2314
2300
|
if not provider and not product_type:
|
|
2315
|
-
return
|
|
2301
|
+
return QueryablesDict(
|
|
2302
|
+
additional_properties=True,
|
|
2303
|
+
**model_fields_to_annotated(CommonQueryables.model_fields),
|
|
2304
|
+
)
|
|
2316
2305
|
|
|
2317
|
-
|
|
2306
|
+
additional_properties = False
|
|
2307
|
+
additional_information = []
|
|
2308
|
+
queryable_properties: dict[str, Any] = {}
|
|
2318
2309
|
|
|
2319
2310
|
for plugin in self._plugins_manager.get_search_plugins(product_type, provider):
|
|
2311
|
+
# attach product type config
|
|
2312
|
+
product_type_configs: dict[str, Any] = {}
|
|
2313
|
+
if product_type:
|
|
2314
|
+
self._attach_product_type_config(plugin, product_type)
|
|
2315
|
+
product_type_configs[product_type] = plugin.config.product_type_config
|
|
2316
|
+
else:
|
|
2317
|
+
for pt in available_product_types:
|
|
2318
|
+
self._attach_product_type_config(plugin, pt)
|
|
2319
|
+
product_type_configs[pt] = plugin.config.product_type_config
|
|
2320
|
+
|
|
2321
|
+
# authenticate if required
|
|
2320
2322
|
if getattr(plugin.config, "need_auth", False) and (
|
|
2321
2323
|
auth := self._plugins_manager.get_auth_plugin(plugin)
|
|
2322
2324
|
):
|
|
2323
|
-
|
|
2324
|
-
|
|
2325
|
-
|
|
2326
|
-
|
|
2327
|
-
|
|
2328
|
-
|
|
2329
|
-
|
|
2330
|
-
)
|
|
2331
|
-
queryables = {
|
|
2332
|
-
k: v
|
|
2333
|
-
for k, v in list(providers_queryables.values())[0].items()
|
|
2334
|
-
if k in queryable_keys
|
|
2335
|
-
}
|
|
2325
|
+
try:
|
|
2326
|
+
plugin.auth = auth.authenticate()
|
|
2327
|
+
except AuthenticationError:
|
|
2328
|
+
logger.debug(
|
|
2329
|
+
"queryables from provider %s could not be fetched due to an authentication error",
|
|
2330
|
+
plugin.provider,
|
|
2331
|
+
)
|
|
2336
2332
|
|
|
2337
|
-
|
|
2338
|
-
|
|
2339
|
-
|
|
2340
|
-
|
|
2341
|
-
|
|
2333
|
+
plugin_queryables = plugin.list_queryables(
|
|
2334
|
+
kwargs,
|
|
2335
|
+
available_product_types,
|
|
2336
|
+
product_type_configs,
|
|
2337
|
+
product_type,
|
|
2338
|
+
pt_alias,
|
|
2339
|
+
)
|
|
2342
2340
|
|
|
2343
|
-
|
|
2341
|
+
if plugin_queryables.additional_information:
|
|
2342
|
+
additional_information.append(
|
|
2343
|
+
f"{plugin.provider}: {plugin_queryables.additional_information}"
|
|
2344
|
+
)
|
|
2345
|
+
queryable_properties = {**plugin_queryables, **queryable_properties}
|
|
2346
|
+
additional_properties = (
|
|
2347
|
+
additional_properties or plugin_queryables.additional_properties
|
|
2348
|
+
)
|
|
2344
2349
|
|
|
2345
|
-
return
|
|
2350
|
+
return QueryablesDict(
|
|
2351
|
+
additional_properties=additional_properties,
|
|
2352
|
+
additional_information=" | ".join(additional_information),
|
|
2353
|
+
**queryable_properties,
|
|
2354
|
+
)
|
|
2346
2355
|
|
|
2347
|
-
def available_sortables(self) ->
|
|
2356
|
+
def available_sortables(self) -> dict[str, Optional[ProviderSortables]]:
|
|
2348
2357
|
"""For each provider, gives its available sortable parameter(s) and its maximum
|
|
2349
2358
|
number of them if it supports the sorting feature, otherwise gives None.
|
|
2350
2359
|
|
|
@@ -2352,7 +2361,7 @@ class EODataAccessGateway:
|
|
|
2352
2361
|
its (their) maximum number as value(s).
|
|
2353
2362
|
:raises: :class:`~eodag.utils.exceptions.UnsupportedProvider`
|
|
2354
2363
|
"""
|
|
2355
|
-
sortables:
|
|
2364
|
+
sortables: dict[str, Optional[ProviderSortables]] = {}
|
|
2356
2365
|
provider_search_plugins = self._plugins_manager.get_search_plugins()
|
|
2357
2366
|
for provider_search_plugin in provider_search_plugins:
|
|
2358
2367
|
provider = provider_search_plugin.provider
|
|
@@ -2375,3 +2384,30 @@ class EODataAccessGateway:
|
|
|
2375
2384
|
],
|
|
2376
2385
|
}
|
|
2377
2386
|
return sortables
|
|
2387
|
+
|
|
2388
|
+
def _attach_product_type_config(self, plugin: Search, product_type: str) -> None:
|
|
2389
|
+
"""
|
|
2390
|
+
Attach product_types_config to plugin config. This dict contains product
|
|
2391
|
+
type metadata that will also be stored in each product's properties.
|
|
2392
|
+
"""
|
|
2393
|
+
try:
|
|
2394
|
+
plugin.config.product_type_config = dict(
|
|
2395
|
+
[
|
|
2396
|
+
p
|
|
2397
|
+
for p in self.list_product_types(
|
|
2398
|
+
plugin.provider, fetch_providers=False
|
|
2399
|
+
)
|
|
2400
|
+
if p["_id"] == product_type
|
|
2401
|
+
][0],
|
|
2402
|
+
**{"productType": product_type},
|
|
2403
|
+
)
|
|
2404
|
+
# If the product isn't in the catalog, it's a generic product type.
|
|
2405
|
+
except IndexError:
|
|
2406
|
+
# Construct the GENERIC_PRODUCT_TYPE metadata
|
|
2407
|
+
plugin.config.product_type_config = dict(
|
|
2408
|
+
ID=GENERIC_PRODUCT_TYPE,
|
|
2409
|
+
**self.product_types_config[GENERIC_PRODUCT_TYPE],
|
|
2410
|
+
productType=product_type,
|
|
2411
|
+
)
|
|
2412
|
+
# Remove the ID since this is equal to productType.
|
|
2413
|
+
plugin.config.product_type_config.pop("ID", None)
|