eodag 3.0.1__py3-none-any.whl → 3.1.0b2__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- eodag/api/core.py +164 -127
- eodag/api/product/_assets.py +11 -11
- eodag/api/product/_product.py +45 -30
- eodag/api/product/drivers/__init__.py +81 -4
- eodag/api/product/drivers/base.py +65 -4
- eodag/api/product/drivers/generic.py +65 -0
- eodag/api/product/drivers/sentinel1.py +97 -0
- eodag/api/product/drivers/sentinel2.py +95 -0
- eodag/api/product/metadata_mapping.py +101 -85
- eodag/api/search_result.py +13 -23
- eodag/cli.py +26 -5
- eodag/config.py +78 -81
- eodag/plugins/apis/base.py +1 -1
- eodag/plugins/apis/ecmwf.py +46 -22
- eodag/plugins/apis/usgs.py +16 -15
- eodag/plugins/authentication/aws_auth.py +16 -13
- eodag/plugins/authentication/base.py +5 -3
- eodag/plugins/authentication/header.py +3 -3
- eodag/plugins/authentication/keycloak.py +4 -4
- eodag/plugins/authentication/oauth.py +7 -3
- eodag/plugins/authentication/openid_connect.py +16 -16
- eodag/plugins/authentication/sas_auth.py +4 -4
- eodag/plugins/authentication/token.py +41 -10
- eodag/plugins/authentication/token_exchange.py +1 -1
- eodag/plugins/base.py +4 -4
- eodag/plugins/crunch/base.py +4 -4
- eodag/plugins/crunch/filter_date.py +4 -4
- eodag/plugins/crunch/filter_latest_intersect.py +6 -6
- eodag/plugins/crunch/filter_latest_tpl_name.py +7 -7
- eodag/plugins/crunch/filter_overlap.py +4 -4
- eodag/plugins/crunch/filter_property.py +6 -7
- eodag/plugins/download/aws.py +58 -78
- eodag/plugins/download/base.py +38 -56
- eodag/plugins/download/creodias_s3.py +29 -0
- eodag/plugins/download/http.py +173 -183
- eodag/plugins/download/s3rest.py +10 -11
- eodag/plugins/manager.py +10 -20
- eodag/plugins/search/__init__.py +6 -5
- eodag/plugins/search/base.py +87 -44
- eodag/plugins/search/build_search_result.py +1067 -329
- eodag/plugins/search/cop_marine.py +22 -12
- eodag/plugins/search/creodias_s3.py +9 -73
- eodag/plugins/search/csw.py +11 -11
- eodag/plugins/search/data_request_search.py +16 -15
- eodag/plugins/search/qssearch.py +103 -187
- eodag/plugins/search/stac_list_assets.py +85 -0
- eodag/plugins/search/static_stac_search.py +3 -3
- eodag/resources/ext_product_types.json +1 -1
- eodag/resources/product_types.yml +663 -304
- eodag/resources/providers.yml +823 -1749
- eodag/resources/stac_api.yml +2 -2
- eodag/resources/user_conf_template.yml +11 -0
- eodag/rest/cache.py +2 -2
- eodag/rest/config.py +3 -3
- eodag/rest/core.py +112 -82
- eodag/rest/errors.py +5 -5
- eodag/rest/server.py +33 -14
- eodag/rest/stac.py +40 -38
- eodag/rest/types/collections_search.py +3 -3
- eodag/rest/types/eodag_search.py +29 -23
- eodag/rest/types/queryables.py +15 -16
- eodag/rest/types/stac_search.py +15 -25
- eodag/rest/utils/__init__.py +14 -21
- eodag/rest/utils/cql_evaluate.py +6 -6
- eodag/rest/utils/rfc3339.py +2 -2
- eodag/types/__init__.py +75 -28
- eodag/types/bbox.py +2 -2
- eodag/types/download_args.py +3 -3
- eodag/types/queryables.py +183 -72
- eodag/types/search_args.py +4 -4
- eodag/types/whoosh.py +127 -3
- eodag/utils/__init__.py +152 -50
- eodag/utils/exceptions.py +28 -21
- eodag/utils/import_system.py +2 -2
- eodag/utils/repr.py +65 -6
- eodag/utils/requests.py +13 -13
- eodag/utils/rest.py +2 -2
- eodag/utils/s3.py +208 -0
- eodag/utils/stac_reader.py +10 -10
- {eodag-3.0.1.dist-info → eodag-3.1.0b2.dist-info}/METADATA +77 -76
- eodag-3.1.0b2.dist-info/RECORD +113 -0
- {eodag-3.0.1.dist-info → eodag-3.1.0b2.dist-info}/WHEEL +1 -1
- {eodag-3.0.1.dist-info → eodag-3.1.0b2.dist-info}/entry_points.txt +4 -2
- eodag/utils/constraints.py +0 -244
- eodag-3.0.1.dist-info/RECORD +0 -109
- {eodag-3.0.1.dist-info → eodag-3.1.0b2.dist-info}/LICENSE +0 -0
- {eodag-3.0.1.dist-info → eodag-3.1.0b2.dist-info}/top_level.txt +0 -0
eodag/api/core.py
CHANGED
|
@@ -24,27 +24,15 @@ import re
|
|
|
24
24
|
import shutil
|
|
25
25
|
import tempfile
|
|
26
26
|
from operator import itemgetter
|
|
27
|
-
from typing import
|
|
28
|
-
TYPE_CHECKING,
|
|
29
|
-
Annotated,
|
|
30
|
-
Any,
|
|
31
|
-
Dict,
|
|
32
|
-
Iterator,
|
|
33
|
-
List,
|
|
34
|
-
Optional,
|
|
35
|
-
Set,
|
|
36
|
-
Tuple,
|
|
37
|
-
Union,
|
|
38
|
-
)
|
|
27
|
+
from typing import TYPE_CHECKING, Any, Iterator, Optional, Union
|
|
39
28
|
|
|
40
29
|
import geojson
|
|
41
30
|
import pkg_resources
|
|
42
31
|
import yaml.parser
|
|
43
32
|
from pkg_resources import resource_filename
|
|
44
|
-
from pydantic.fields import FieldInfo
|
|
45
33
|
from whoosh import analysis, fields
|
|
46
34
|
from whoosh.fields import Schema
|
|
47
|
-
from whoosh.index import
|
|
35
|
+
from whoosh.index import exists_in, open_dir
|
|
48
36
|
from whoosh.qparser import QueryParser
|
|
49
37
|
|
|
50
38
|
from eodag.api.product.metadata_mapping import (
|
|
@@ -69,11 +57,11 @@ from eodag.config import (
|
|
|
69
57
|
)
|
|
70
58
|
from eodag.plugins.manager import PluginManager
|
|
71
59
|
from eodag.plugins.search import PreparedSearch
|
|
72
|
-
from eodag.plugins.search.build_search_result import
|
|
60
|
+
from eodag.plugins.search.build_search_result import MeteoblueSearch
|
|
73
61
|
from eodag.plugins.search.qssearch import PostJsonSearch
|
|
74
62
|
from eodag.types import model_fields_to_annotated
|
|
75
|
-
from eodag.types.queryables import CommonQueryables
|
|
76
|
-
from eodag.types.whoosh import EODAGQueryParser
|
|
63
|
+
from eodag.types.queryables import CommonQueryables, QueryablesDict
|
|
64
|
+
from eodag.types.whoosh import EODAGQueryParser, create_in
|
|
77
65
|
from eodag.utils import (
|
|
78
66
|
DEFAULT_DOWNLOAD_TIMEOUT,
|
|
79
67
|
DEFAULT_DOWNLOAD_WAIT,
|
|
@@ -84,7 +72,6 @@ from eodag.utils import (
|
|
|
84
72
|
HTTP_REQ_TIMEOUT,
|
|
85
73
|
MockResponse,
|
|
86
74
|
_deprecated,
|
|
87
|
-
copy_deepcopy,
|
|
88
75
|
get_geometry_from_various,
|
|
89
76
|
makedirs,
|
|
90
77
|
obj_md5sum,
|
|
@@ -93,6 +80,7 @@ from eodag.utils import (
|
|
|
93
80
|
uri_to_path,
|
|
94
81
|
)
|
|
95
82
|
from eodag.utils.exceptions import (
|
|
83
|
+
AuthenticationError,
|
|
96
84
|
EodagError,
|
|
97
85
|
NoMatchingProductType,
|
|
98
86
|
PluginImplementationError,
|
|
@@ -197,7 +185,7 @@ class EODataAccessGateway:
|
|
|
197
185
|
self._plugins_manager.rebuild(self.providers_config)
|
|
198
186
|
|
|
199
187
|
# store pruned providers configs
|
|
200
|
-
self._pruned_providers_config:
|
|
188
|
+
self._pruned_providers_config: dict[str, Any] = {}
|
|
201
189
|
# filter out providers needing auth that have no credentials set
|
|
202
190
|
self._prune_providers_list()
|
|
203
191
|
|
|
@@ -219,9 +207,10 @@ class EODataAccessGateway:
|
|
|
219
207
|
"eodag",
|
|
220
208
|
os.path.join("resources", "locations_conf_template.yml"),
|
|
221
209
|
)
|
|
222
|
-
with
|
|
223
|
-
|
|
224
|
-
|
|
210
|
+
with (
|
|
211
|
+
open(locations_conf_template) as infile,
|
|
212
|
+
open(locations_conf_path, "w") as outfile,
|
|
213
|
+
):
|
|
225
214
|
# The template contains paths in the form of:
|
|
226
215
|
# /path/to/locations/file.shp
|
|
227
216
|
path_template = "/path/to/locations/"
|
|
@@ -317,13 +306,18 @@ class EODataAccessGateway:
|
|
|
317
306
|
product_type, **{"md5": self.product_types_config_md5}
|
|
318
307
|
)
|
|
319
308
|
# add to index
|
|
320
|
-
|
|
321
|
-
|
|
322
|
-
|
|
323
|
-
|
|
324
|
-
|
|
325
|
-
|
|
326
|
-
|
|
309
|
+
try:
|
|
310
|
+
ix_writer.add_document(
|
|
311
|
+
**{
|
|
312
|
+
k: v
|
|
313
|
+
for k, v in versioned_product_type.items()
|
|
314
|
+
if k in product_types_schema.names()
|
|
315
|
+
}
|
|
316
|
+
)
|
|
317
|
+
except TypeError as e:
|
|
318
|
+
logger.error(
|
|
319
|
+
f"Cannot write product type {product_type['ID']} into index. e={e} product_type={product_type}"
|
|
320
|
+
)
|
|
327
321
|
ix_writer.commit()
|
|
328
322
|
|
|
329
323
|
def set_preferred_provider(self, provider: str) -> None:
|
|
@@ -341,7 +335,7 @@ class EODataAccessGateway:
|
|
|
341
335
|
new_priority = max_priority + 1
|
|
342
336
|
self._plugins_manager.set_priority(provider, new_priority)
|
|
343
337
|
|
|
344
|
-
def get_preferred_provider(self) ->
|
|
338
|
+
def get_preferred_provider(self) -> tuple[str, int]:
|
|
345
339
|
"""Get the provider currently set as the preferred one for searching
|
|
346
340
|
products, along with its priority.
|
|
347
341
|
|
|
@@ -357,7 +351,7 @@ class EODataAccessGateway:
|
|
|
357
351
|
def update_providers_config(
|
|
358
352
|
self,
|
|
359
353
|
yaml_conf: Optional[str] = None,
|
|
360
|
-
dict_conf: Optional[
|
|
354
|
+
dict_conf: Optional[dict[str, Any]] = None,
|
|
361
355
|
) -> None:
|
|
362
356
|
"""Update providers configuration with given input.
|
|
363
357
|
Can be used to add a provider to existing configuration or update
|
|
@@ -403,12 +397,12 @@ class EODataAccessGateway:
|
|
|
403
397
|
name: str,
|
|
404
398
|
url: Optional[str] = None,
|
|
405
399
|
priority: Optional[int] = None,
|
|
406
|
-
search:
|
|
407
|
-
products:
|
|
400
|
+
search: dict[str, Any] = {"type": "StacSearch"},
|
|
401
|
+
products: dict[str, Any] = {
|
|
408
402
|
GENERIC_PRODUCT_TYPE: {"productType": "{productType}"}
|
|
409
403
|
},
|
|
410
|
-
download:
|
|
411
|
-
**kwargs:
|
|
404
|
+
download: dict[str, Any] = {"type": "HTTPDownload", "auth_error_code": 401},
|
|
405
|
+
**kwargs: dict[str, Any],
|
|
412
406
|
):
|
|
413
407
|
"""Adds a new provider.
|
|
414
408
|
|
|
@@ -427,7 +421,7 @@ class EODataAccessGateway:
|
|
|
427
421
|
:param download: Download :class:`~eodag.config.PluginConfig` mapping
|
|
428
422
|
:param kwargs: Additional :class:`~eodag.config.ProviderConfig` mapping
|
|
429
423
|
"""
|
|
430
|
-
conf_dict:
|
|
424
|
+
conf_dict: dict[str, Any] = {
|
|
431
425
|
name: {
|
|
432
426
|
"url": url,
|
|
433
427
|
"search": {"type": "StacSearch", **search},
|
|
@@ -571,7 +565,7 @@ class EODataAccessGateway:
|
|
|
571
565
|
main_locations_config = locations_config[main_key]
|
|
572
566
|
|
|
573
567
|
logger.info("Locations configuration loaded from %s" % locations_conf_path)
|
|
574
|
-
self.locations_config:
|
|
568
|
+
self.locations_config: list[dict[str, Any]] = main_locations_config
|
|
575
569
|
else:
|
|
576
570
|
logger.info(
|
|
577
571
|
"Could not load locations configuration from %s" % locations_conf_path
|
|
@@ -580,7 +574,7 @@ class EODataAccessGateway:
|
|
|
580
574
|
|
|
581
575
|
def list_product_types(
|
|
582
576
|
self, provider: Optional[str] = None, fetch_providers: bool = True
|
|
583
|
-
) ->
|
|
577
|
+
) -> list[dict[str, Any]]:
|
|
584
578
|
"""Lists supported product types.
|
|
585
579
|
|
|
586
580
|
:param provider: (optional) The name of a provider that must support the product
|
|
@@ -594,7 +588,7 @@ class EODataAccessGateway:
|
|
|
594
588
|
# First, update product types list if possible
|
|
595
589
|
self.fetch_product_types_list(provider=provider)
|
|
596
590
|
|
|
597
|
-
product_types:
|
|
591
|
+
product_types: list[dict[str, Any]] = []
|
|
598
592
|
|
|
599
593
|
providers_configs = (
|
|
600
594
|
list(self.providers_config.values())
|
|
@@ -650,7 +644,7 @@ class EODataAccessGateway:
|
|
|
650
644
|
providers_to_fetch = [provider]
|
|
651
645
|
|
|
652
646
|
# providers discovery confs that are fetchable
|
|
653
|
-
providers_discovery_configs_fetchable:
|
|
647
|
+
providers_discovery_configs_fetchable: dict[str, Any] = {}
|
|
654
648
|
# check if any provider has not already been fetched for product types
|
|
655
649
|
already_fetched = True
|
|
656
650
|
for provider_to_fetch in providers_to_fetch:
|
|
@@ -773,7 +767,7 @@ class EODataAccessGateway:
|
|
|
773
767
|
|
|
774
768
|
def discover_product_types(
|
|
775
769
|
self, provider: Optional[str] = None
|
|
776
|
-
) -> Optional[
|
|
770
|
+
) -> Optional[dict[str, Any]]:
|
|
777
771
|
"""Fetch providers for product types
|
|
778
772
|
|
|
779
773
|
:param provider: The name of a provider or provider-group to fetch. Defaults to
|
|
@@ -793,7 +787,7 @@ class EODataAccessGateway:
|
|
|
793
787
|
raise UnsupportedProvider(
|
|
794
788
|
f"The requested provider is not (yet) supported: {provider}"
|
|
795
789
|
)
|
|
796
|
-
ext_product_types_conf:
|
|
790
|
+
ext_product_types_conf: dict[str, Any] = {}
|
|
797
791
|
providers_to_fetch = [
|
|
798
792
|
p
|
|
799
793
|
for p in (
|
|
@@ -806,7 +800,7 @@ class EODataAccessGateway:
|
|
|
806
800
|
else self.available_providers()
|
|
807
801
|
)
|
|
808
802
|
]
|
|
809
|
-
kwargs:
|
|
803
|
+
kwargs: dict[str, Any] = {}
|
|
810
804
|
for provider in providers_to_fetch:
|
|
811
805
|
if hasattr(self.providers_config[provider], "search"):
|
|
812
806
|
search_plugin_config = self.providers_config[provider].search
|
|
@@ -847,7 +841,7 @@ class EODataAccessGateway:
|
|
|
847
841
|
return sort_dict(ext_product_types_conf)
|
|
848
842
|
|
|
849
843
|
def update_product_types_list(
|
|
850
|
-
self, ext_product_types_conf:
|
|
844
|
+
self, ext_product_types_conf: dict[str, Optional[dict[str, dict[str, Any]]]]
|
|
851
845
|
) -> None:
|
|
852
846
|
"""Update eodag product types list
|
|
853
847
|
|
|
@@ -875,7 +869,7 @@ class EODataAccessGateway:
|
|
|
875
869
|
provider,
|
|
876
870
|
)
|
|
877
871
|
continue
|
|
878
|
-
new_product_types:
|
|
872
|
+
new_product_types: list[str] = []
|
|
879
873
|
for (
|
|
880
874
|
new_product_type,
|
|
881
875
|
new_product_type_conf,
|
|
@@ -938,7 +932,7 @@ class EODataAccessGateway:
|
|
|
938
932
|
|
|
939
933
|
def available_providers(
|
|
940
934
|
self, product_type: Optional[str] = None, by_group: bool = False
|
|
941
|
-
) ->
|
|
935
|
+
) -> list[str]:
|
|
942
936
|
"""Gives the sorted list of the available providers or groups
|
|
943
937
|
|
|
944
938
|
The providers or groups are sorted first by their priority level in descending order,
|
|
@@ -965,7 +959,7 @@ class EODataAccessGateway:
|
|
|
965
959
|
|
|
966
960
|
# If by_group is True, keep only the highest priority for each group
|
|
967
961
|
if by_group:
|
|
968
|
-
group_priority:
|
|
962
|
+
group_priority: dict[str, int] = {}
|
|
969
963
|
for name, priority in providers:
|
|
970
964
|
if name not in group_priority or priority > group_priority[name]:
|
|
971
965
|
group_priority[name] = priority
|
|
@@ -1032,7 +1026,7 @@ class EODataAccessGateway:
|
|
|
1032
1026
|
missionStartDate: Optional[str] = None,
|
|
1033
1027
|
missionEndDate: Optional[str] = None,
|
|
1034
1028
|
**kwargs: Any,
|
|
1035
|
-
) ->
|
|
1029
|
+
) -> list[str]:
|
|
1036
1030
|
"""
|
|
1037
1031
|
Find EODAG product type IDs that best match a set of search parameters.
|
|
1038
1032
|
|
|
@@ -1090,7 +1084,7 @@ class EODataAccessGateway:
|
|
|
1090
1084
|
query = p.parse(text)
|
|
1091
1085
|
results = searcher.search(query, limit=None)
|
|
1092
1086
|
|
|
1093
|
-
guesses:
|
|
1087
|
+
guesses: list[dict[str, str]] = [dict(r) for r in results or []]
|
|
1094
1088
|
|
|
1095
1089
|
# datetime filtering
|
|
1096
1090
|
if missionStartDate or missionEndDate:
|
|
@@ -1131,8 +1125,8 @@ class EODataAccessGateway:
|
|
|
1131
1125
|
raise_errors: bool = False,
|
|
1132
1126
|
start: Optional[str] = None,
|
|
1133
1127
|
end: Optional[str] = None,
|
|
1134
|
-
geom: Optional[Union[str,
|
|
1135
|
-
locations: Optional[
|
|
1128
|
+
geom: Optional[Union[str, dict[str, float], BaseGeometry]] = None,
|
|
1129
|
+
locations: Optional[dict[str, str]] = None,
|
|
1136
1130
|
provider: Optional[str] = None,
|
|
1137
1131
|
count: bool = False,
|
|
1138
1132
|
**kwargs: Any,
|
|
@@ -1211,7 +1205,7 @@ class EODataAccessGateway:
|
|
|
1211
1205
|
items_per_page=items_per_page,
|
|
1212
1206
|
)
|
|
1213
1207
|
|
|
1214
|
-
errors:
|
|
1208
|
+
errors: list[tuple[str, Exception]] = []
|
|
1215
1209
|
# Loop over available providers and return the first non-empty results
|
|
1216
1210
|
for i, search_plugin in enumerate(search_plugins):
|
|
1217
1211
|
search_plugin.clear()
|
|
@@ -1240,8 +1234,8 @@ class EODataAccessGateway:
|
|
|
1240
1234
|
items_per_page: int = DEFAULT_ITEMS_PER_PAGE,
|
|
1241
1235
|
start: Optional[str] = None,
|
|
1242
1236
|
end: Optional[str] = None,
|
|
1243
|
-
geom: Optional[Union[str,
|
|
1244
|
-
locations: Optional[
|
|
1237
|
+
geom: Optional[Union[str, dict[str, float], BaseGeometry]] = None,
|
|
1238
|
+
locations: Optional[dict[str, str]] = None,
|
|
1245
1239
|
**kwargs: Any,
|
|
1246
1240
|
) -> Iterator[SearchResult]:
|
|
1247
1241
|
"""Iterate over the pages of a products search.
|
|
@@ -1417,8 +1411,8 @@ class EODataAccessGateway:
|
|
|
1417
1411
|
items_per_page: Optional[int] = None,
|
|
1418
1412
|
start: Optional[str] = None,
|
|
1419
1413
|
end: Optional[str] = None,
|
|
1420
|
-
geom: Optional[Union[str,
|
|
1421
|
-
locations: Optional[
|
|
1414
|
+
geom: Optional[Union[str, dict[str, float], BaseGeometry]] = None,
|
|
1415
|
+
locations: Optional[dict[str, str]] = None,
|
|
1422
1416
|
**kwargs: Any,
|
|
1423
1417
|
) -> SearchResult:
|
|
1424
1418
|
"""Search and return all the products matching the search criteria.
|
|
@@ -1639,7 +1633,7 @@ class EODataAccessGateway:
|
|
|
1639
1633
|
if not getattr(plugin.config, "discover_product_types", {}).get("fetch_url"):
|
|
1640
1634
|
return None
|
|
1641
1635
|
|
|
1642
|
-
kwargs:
|
|
1636
|
+
kwargs: dict[str, Any] = {"productType": product_type}
|
|
1643
1637
|
|
|
1644
1638
|
# append auth if needed
|
|
1645
1639
|
if getattr(plugin.config, "need_auth", False):
|
|
@@ -1657,11 +1651,11 @@ class EODataAccessGateway:
|
|
|
1657
1651
|
self,
|
|
1658
1652
|
start: Optional[str] = None,
|
|
1659
1653
|
end: Optional[str] = None,
|
|
1660
|
-
geom: Optional[Union[str,
|
|
1661
|
-
locations: Optional[
|
|
1654
|
+
geom: Optional[Union[str, dict[str, float], BaseGeometry]] = None,
|
|
1655
|
+
locations: Optional[dict[str, str]] = None,
|
|
1662
1656
|
provider: Optional[str] = None,
|
|
1663
1657
|
**kwargs: Any,
|
|
1664
|
-
) ->
|
|
1658
|
+
) -> tuple[list[Union[Search, Api]], dict[str, Any]]:
|
|
1665
1659
|
"""Internal method to prepare the search kwargs and get the search plugins.
|
|
1666
1660
|
|
|
1667
1661
|
Product query:
|
|
@@ -1769,16 +1763,16 @@ class EODataAccessGateway:
|
|
|
1769
1763
|
|
|
1770
1764
|
preferred_provider = self.get_preferred_provider()[0]
|
|
1771
1765
|
|
|
1772
|
-
search_plugins:
|
|
1766
|
+
search_plugins: list[Union[Search, Api]] = []
|
|
1773
1767
|
for plugin in self._plugins_manager.get_search_plugins(
|
|
1774
1768
|
product_type=product_type, provider=provider
|
|
1775
1769
|
):
|
|
1776
|
-
# exclude
|
|
1770
|
+
# exclude MeteoblueSearch plugins from search fallback for unknown product_type
|
|
1777
1771
|
if (
|
|
1778
1772
|
provider != plugin.provider
|
|
1779
1773
|
and preferred_provider != plugin.provider
|
|
1780
1774
|
and product_type not in self.product_types_config
|
|
1781
|
-
and isinstance(plugin,
|
|
1775
|
+
and isinstance(plugin, MeteoblueSearch)
|
|
1782
1776
|
):
|
|
1783
1777
|
continue
|
|
1784
1778
|
search_plugins.append(plugin)
|
|
@@ -1801,27 +1795,7 @@ class EODataAccessGateway:
|
|
|
1801
1795
|
# Add product_types_config to plugin config. This dict contains product
|
|
1802
1796
|
# type metadata that will also be stored in each product's properties.
|
|
1803
1797
|
for search_plugin in search_plugins:
|
|
1804
|
-
|
|
1805
|
-
search_plugin.config.product_type_config = dict(
|
|
1806
|
-
[
|
|
1807
|
-
p
|
|
1808
|
-
for p in self.list_product_types(
|
|
1809
|
-
search_plugin.provider, fetch_providers=False
|
|
1810
|
-
)
|
|
1811
|
-
if p["_id"] == product_type
|
|
1812
|
-
][0],
|
|
1813
|
-
**{"productType": product_type},
|
|
1814
|
-
)
|
|
1815
|
-
# If the product isn't in the catalog, it's a generic product type.
|
|
1816
|
-
except IndexError:
|
|
1817
|
-
# Construct the GENERIC_PRODUCT_TYPE metadata
|
|
1818
|
-
search_plugin.config.product_type_config = dict(
|
|
1819
|
-
ID=GENERIC_PRODUCT_TYPE,
|
|
1820
|
-
**self.product_types_config[GENERIC_PRODUCT_TYPE],
|
|
1821
|
-
productType=product_type,
|
|
1822
|
-
)
|
|
1823
|
-
# Remove the ID since this is equal to productType.
|
|
1824
|
-
search_plugin.config.product_type_config.pop("ID", None)
|
|
1798
|
+
self._attach_product_type_config(search_plugin, product_type)
|
|
1825
1799
|
|
|
1826
1800
|
return search_plugins, kwargs
|
|
1827
1801
|
|
|
@@ -1859,10 +1833,10 @@ class EODataAccessGateway:
|
|
|
1859
1833
|
max_items_per_page,
|
|
1860
1834
|
)
|
|
1861
1835
|
|
|
1862
|
-
results:
|
|
1836
|
+
results: list[EOProduct] = []
|
|
1863
1837
|
total_results: Optional[int] = 0 if count else None
|
|
1864
1838
|
|
|
1865
|
-
errors:
|
|
1839
|
+
errors: list[tuple[str, Exception]] = []
|
|
1866
1840
|
|
|
1867
1841
|
try:
|
|
1868
1842
|
prep = PreparedSearch(count=count)
|
|
@@ -2010,7 +1984,7 @@ class EODataAccessGateway:
|
|
|
2010
1984
|
return results
|
|
2011
1985
|
|
|
2012
1986
|
@staticmethod
|
|
2013
|
-
def group_by_extent(searches:
|
|
1987
|
+
def group_by_extent(searches: list[SearchResult]) -> list[SearchResult]:
|
|
2014
1988
|
"""Combines multiple SearchResults and return a list of SearchResults grouped
|
|
2015
1989
|
by extent (i.e. bounding box).
|
|
2016
1990
|
|
|
@@ -2019,7 +1993,7 @@ class EODataAccessGateway:
|
|
|
2019
1993
|
"""
|
|
2020
1994
|
# Dict with extents as keys, each extent being defined by a str
|
|
2021
1995
|
# "{minx}{miny}{maxx}{maxy}" (each float rounded to 2 dec).
|
|
2022
|
-
products_grouped_by_extent:
|
|
1996
|
+
products_grouped_by_extent: dict[str, Any] = {}
|
|
2023
1997
|
|
|
2024
1998
|
for search in searches:
|
|
2025
1999
|
for product in search:
|
|
@@ -2038,10 +2012,10 @@ class EODataAccessGateway:
|
|
|
2038
2012
|
search_result: SearchResult,
|
|
2039
2013
|
downloaded_callback: Optional[DownloadedCallback] = None,
|
|
2040
2014
|
progress_callback: Optional[ProgressCallback] = None,
|
|
2041
|
-
wait:
|
|
2042
|
-
timeout:
|
|
2015
|
+
wait: float = DEFAULT_DOWNLOAD_WAIT,
|
|
2016
|
+
timeout: float = DEFAULT_DOWNLOAD_TIMEOUT,
|
|
2043
2017
|
**kwargs: Unpack[DownloadConf],
|
|
2044
|
-
) ->
|
|
2018
|
+
) -> list[str]:
|
|
2045
2019
|
"""Download all products resulting from a search.
|
|
2046
2020
|
|
|
2047
2021
|
:param search_result: A collection of EO products resulting from a search
|
|
@@ -2201,8 +2175,8 @@ class EODataAccessGateway:
|
|
|
2201
2175
|
self,
|
|
2202
2176
|
product: EOProduct,
|
|
2203
2177
|
progress_callback: Optional[ProgressCallback] = None,
|
|
2204
|
-
wait:
|
|
2205
|
-
timeout:
|
|
2178
|
+
wait: float = DEFAULT_DOWNLOAD_WAIT,
|
|
2179
|
+
timeout: float = DEFAULT_DOWNLOAD_TIMEOUT,
|
|
2206
2180
|
**kwargs: Unpack[DownloadConf],
|
|
2207
2181
|
) -> str:
|
|
2208
2182
|
"""Download a single product.
|
|
@@ -2280,71 +2254,107 @@ class EODataAccessGateway:
|
|
|
2280
2254
|
return self._plugins_manager.get_crunch_plugin(name, **plugin_conf)
|
|
2281
2255
|
|
|
2282
2256
|
def list_queryables(
|
|
2283
|
-
self,
|
|
2284
|
-
|
|
2257
|
+
self,
|
|
2258
|
+
provider: Optional[str] = None,
|
|
2259
|
+
fetch_providers: bool = True,
|
|
2260
|
+
**kwargs: Any,
|
|
2261
|
+
) -> QueryablesDict:
|
|
2285
2262
|
"""Fetch the queryable properties for a given product type and/or provider.
|
|
2286
2263
|
|
|
2287
2264
|
:param provider: (optional) The provider.
|
|
2265
|
+
:param fetch_providers: If new product types should be fetched from the providers; default: True
|
|
2288
2266
|
:param kwargs: additional filters for queryables (`productType` or other search
|
|
2289
2267
|
arguments)
|
|
2290
2268
|
|
|
2291
2269
|
:raises UnsupportedProductType: If the specified product type is not available for the
|
|
2292
2270
|
provider.
|
|
2293
2271
|
|
|
2294
|
-
:returns: A
|
|
2295
|
-
parameters to their annotated type
|
|
2272
|
+
:returns: A :class:`~eodag.api.product.queryables.QuerybalesDict` containing the EODAG queryable
|
|
2273
|
+
properties, associating parameters to their annotated type, and a additional_properties attribute
|
|
2296
2274
|
"""
|
|
2297
|
-
|
|
2275
|
+
# only fetch providers if product type is not found
|
|
2276
|
+
available_product_types: list[str] = [
|
|
2298
2277
|
pt["ID"]
|
|
2299
2278
|
for pt in self.list_product_types(provider=provider, fetch_providers=False)
|
|
2300
2279
|
]
|
|
2301
|
-
product_type = kwargs.get("productType")
|
|
2280
|
+
product_type: Optional[str] = kwargs.get("productType")
|
|
2281
|
+
pt_alias: Optional[str] = product_type
|
|
2302
2282
|
|
|
2303
2283
|
if product_type:
|
|
2284
|
+
if product_type not in available_product_types:
|
|
2285
|
+
if fetch_providers:
|
|
2286
|
+
# fetch providers and try again
|
|
2287
|
+
available_product_types = [
|
|
2288
|
+
pt["ID"]
|
|
2289
|
+
for pt in self.list_product_types(
|
|
2290
|
+
provider=provider, fetch_providers=True
|
|
2291
|
+
)
|
|
2292
|
+
]
|
|
2293
|
+
raise UnsupportedProductType(f"{product_type} is not available.")
|
|
2304
2294
|
try:
|
|
2305
2295
|
kwargs["productType"] = product_type = self.get_product_type_from_alias(
|
|
2306
2296
|
product_type
|
|
2307
2297
|
)
|
|
2308
2298
|
except NoMatchingProductType as e:
|
|
2309
|
-
raise UnsupportedProductType(f"{product_type} is not available") from e
|
|
2310
|
-
|
|
2311
|
-
if product_type and product_type not in available_product_types:
|
|
2312
|
-
self.fetch_product_types_list()
|
|
2299
|
+
raise UnsupportedProductType(f"{product_type} is not available.") from e
|
|
2313
2300
|
|
|
2314
2301
|
if not provider and not product_type:
|
|
2315
|
-
return
|
|
2302
|
+
return QueryablesDict(
|
|
2303
|
+
additional_properties=True,
|
|
2304
|
+
**model_fields_to_annotated(CommonQueryables.model_fields),
|
|
2305
|
+
)
|
|
2316
2306
|
|
|
2317
|
-
|
|
2307
|
+
additional_properties = False
|
|
2308
|
+
additional_information = []
|
|
2309
|
+
queryable_properties: dict[str, Any] = {}
|
|
2318
2310
|
|
|
2319
2311
|
for plugin in self._plugins_manager.get_search_plugins(product_type, provider):
|
|
2312
|
+
# attach product type config
|
|
2313
|
+
product_type_configs: dict[str, Any] = {}
|
|
2314
|
+
if product_type:
|
|
2315
|
+
self._attach_product_type_config(plugin, product_type)
|
|
2316
|
+
product_type_configs[product_type] = plugin.config.product_type_config
|
|
2317
|
+
else:
|
|
2318
|
+
for pt in available_product_types:
|
|
2319
|
+
self._attach_product_type_config(plugin, pt)
|
|
2320
|
+
product_type_configs[pt] = plugin.config.product_type_config
|
|
2321
|
+
|
|
2322
|
+
# authenticate if required
|
|
2320
2323
|
if getattr(plugin.config, "need_auth", False) and (
|
|
2321
2324
|
auth := self._plugins_manager.get_auth_plugin(plugin)
|
|
2322
2325
|
):
|
|
2323
|
-
|
|
2324
|
-
|
|
2325
|
-
|
|
2326
|
-
|
|
2327
|
-
|
|
2328
|
-
|
|
2329
|
-
|
|
2330
|
-
)
|
|
2331
|
-
queryables = {
|
|
2332
|
-
k: v
|
|
2333
|
-
for k, v in list(providers_queryables.values())[0].items()
|
|
2334
|
-
if k in queryable_keys
|
|
2335
|
-
}
|
|
2326
|
+
try:
|
|
2327
|
+
plugin.auth = auth.authenticate()
|
|
2328
|
+
except AuthenticationError:
|
|
2329
|
+
logger.debug(
|
|
2330
|
+
"queryables from provider %s could not be fetched due to an authentication error",
|
|
2331
|
+
plugin.provider,
|
|
2332
|
+
)
|
|
2336
2333
|
|
|
2337
|
-
|
|
2338
|
-
|
|
2339
|
-
|
|
2340
|
-
|
|
2341
|
-
|
|
2334
|
+
plugin_queryables = plugin.list_queryables(
|
|
2335
|
+
kwargs,
|
|
2336
|
+
available_product_types,
|
|
2337
|
+
product_type_configs,
|
|
2338
|
+
product_type,
|
|
2339
|
+
pt_alias,
|
|
2340
|
+
)
|
|
2342
2341
|
|
|
2343
|
-
|
|
2342
|
+
if plugin_queryables.additional_information:
|
|
2343
|
+
additional_information.append(
|
|
2344
|
+
f"{plugin.provider}: {plugin_queryables.additional_information}"
|
|
2345
|
+
)
|
|
2346
|
+
queryable_properties = {**plugin_queryables, **queryable_properties}
|
|
2347
|
+
additional_properties = (
|
|
2348
|
+
additional_properties or plugin_queryables.additional_properties
|
|
2349
|
+
)
|
|
2344
2350
|
|
|
2345
|
-
return
|
|
2351
|
+
return QueryablesDict(
|
|
2352
|
+
additional_properties=additional_properties,
|
|
2353
|
+
additional_information=" | ".join(additional_information),
|
|
2354
|
+
**queryable_properties,
|
|
2355
|
+
)
|
|
2346
2356
|
|
|
2347
|
-
def available_sortables(self) ->
|
|
2357
|
+
def available_sortables(self) -> dict[str, Optional[ProviderSortables]]:
|
|
2348
2358
|
"""For each provider, gives its available sortable parameter(s) and its maximum
|
|
2349
2359
|
number of them if it supports the sorting feature, otherwise gives None.
|
|
2350
2360
|
|
|
@@ -2352,7 +2362,7 @@ class EODataAccessGateway:
|
|
|
2352
2362
|
its (their) maximum number as value(s).
|
|
2353
2363
|
:raises: :class:`~eodag.utils.exceptions.UnsupportedProvider`
|
|
2354
2364
|
"""
|
|
2355
|
-
sortables:
|
|
2365
|
+
sortables: dict[str, Optional[ProviderSortables]] = {}
|
|
2356
2366
|
provider_search_plugins = self._plugins_manager.get_search_plugins()
|
|
2357
2367
|
for provider_search_plugin in provider_search_plugins:
|
|
2358
2368
|
provider = provider_search_plugin.provider
|
|
@@ -2375,3 +2385,30 @@ class EODataAccessGateway:
|
|
|
2375
2385
|
],
|
|
2376
2386
|
}
|
|
2377
2387
|
return sortables
|
|
2388
|
+
|
|
2389
|
+
def _attach_product_type_config(self, plugin: Search, product_type: str) -> None:
|
|
2390
|
+
"""
|
|
2391
|
+
Attach product_types_config to plugin config. This dict contains product
|
|
2392
|
+
type metadata that will also be stored in each product's properties.
|
|
2393
|
+
"""
|
|
2394
|
+
try:
|
|
2395
|
+
plugin.config.product_type_config = dict(
|
|
2396
|
+
[
|
|
2397
|
+
p
|
|
2398
|
+
for p in self.list_product_types(
|
|
2399
|
+
plugin.provider, fetch_providers=False
|
|
2400
|
+
)
|
|
2401
|
+
if p["_id"] == product_type
|
|
2402
|
+
][0],
|
|
2403
|
+
**{"productType": product_type},
|
|
2404
|
+
)
|
|
2405
|
+
# If the product isn't in the catalog, it's a generic product type.
|
|
2406
|
+
except IndexError:
|
|
2407
|
+
# Construct the GENERIC_PRODUCT_TYPE metadata
|
|
2408
|
+
plugin.config.product_type_config = dict(
|
|
2409
|
+
ID=GENERIC_PRODUCT_TYPE,
|
|
2410
|
+
**self.product_types_config[GENERIC_PRODUCT_TYPE],
|
|
2411
|
+
productType=product_type,
|
|
2412
|
+
)
|
|
2413
|
+
# Remove the ID since this is equal to productType.
|
|
2414
|
+
plugin.config.product_type_config.pop("ID", None)
|
eodag/api/product/_assets.py
CHANGED
|
@@ -19,7 +19,7 @@ from __future__ import annotations
|
|
|
19
19
|
|
|
20
20
|
import re
|
|
21
21
|
from collections import UserDict
|
|
22
|
-
from typing import TYPE_CHECKING, Any,
|
|
22
|
+
from typing import TYPE_CHECKING, Any, Optional
|
|
23
23
|
|
|
24
24
|
from eodag.utils.exceptions import NotAvailableError
|
|
25
25
|
from eodag.utils.repr import dict_to_html_table
|
|
@@ -45,10 +45,10 @@ class AssetsDict(UserDict):
|
|
|
45
45
|
self.product = product
|
|
46
46
|
super(AssetsDict, self).__init__(*args, **kwargs)
|
|
47
47
|
|
|
48
|
-
def __setitem__(self, key: str, value:
|
|
48
|
+
def __setitem__(self, key: str, value: dict[str, Any]) -> None:
|
|
49
49
|
super().__setitem__(key, Asset(self.product, key, value))
|
|
50
50
|
|
|
51
|
-
def as_dict(self) ->
|
|
51
|
+
def as_dict(self) -> dict[str, Any]:
|
|
52
52
|
"""Builds a representation of AssetsDict to enable its serialization
|
|
53
53
|
|
|
54
54
|
:returns: The representation of a :class:`~eodag.api.product._assets.AssetsDict`
|
|
@@ -56,7 +56,7 @@ class AssetsDict(UserDict):
|
|
|
56
56
|
"""
|
|
57
57
|
return {k: v.as_dict() for k, v in self.data.items()}
|
|
58
58
|
|
|
59
|
-
def get_values(self, asset_filter: str = "") ->
|
|
59
|
+
def get_values(self, asset_filter: str = "") -> list[Asset]:
|
|
60
60
|
"""
|
|
61
61
|
retrieves the assets matching the given filter
|
|
62
62
|
|
|
@@ -98,12 +98,12 @@ class AssetsDict(UserDict):
|
|
|
98
98
|
<details><summary style='color: grey;'>
|
|
99
99
|
<span style='color: black'>'{k}'</span>: 
|
|
100
100
|
{{
|
|
101
|
-
{"'roles': '<span style='color: black'>"+str(v['roles'])+"</span>', "
|
|
102
|
-
|
|
103
|
-
{"'type': '"+str(v['type'])+"', "
|
|
104
|
-
|
|
105
|
-
{"'title': '<span style='color: black'>"+str(v['title'])+"</span>', "
|
|
106
|
-
|
|
101
|
+
{"'roles': '<span style='color: black'>" + str(v['roles']) + "</span>', "
|
|
102
|
+
if v.get("roles") else ""}
|
|
103
|
+
{"'type': '" + str(v['type']) + "', "
|
|
104
|
+
if v.get("type") else ""}
|
|
105
|
+
{"'title': '<span style='color: black'>" + str(v['title']) + "</span>', "
|
|
106
|
+
if v.get("title") else ""}
|
|
107
107
|
...
|
|
108
108
|
}}
|
|
109
109
|
</summary>
|
|
@@ -138,7 +138,7 @@ class Asset(UserDict):
|
|
|
138
138
|
self.key = key
|
|
139
139
|
super(Asset, self).__init__(*args, **kwargs)
|
|
140
140
|
|
|
141
|
-
def as_dict(self) ->
|
|
141
|
+
def as_dict(self) -> dict[str, Any]:
|
|
142
142
|
"""Builds a representation of Asset to enable its serialization
|
|
143
143
|
|
|
144
144
|
:returns: The representation of a :class:`~eodag.api.product._assets.Asset` as a
|