eodag 3.0.0b3__py3-none-any.whl → 3.0.1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- eodag/api/core.py +189 -125
- eodag/api/product/metadata_mapping.py +12 -3
- eodag/api/search_result.py +29 -3
- eodag/cli.py +35 -19
- eodag/config.py +412 -116
- eodag/plugins/apis/base.py +10 -4
- eodag/plugins/apis/ecmwf.py +14 -4
- eodag/plugins/apis/usgs.py +25 -2
- eodag/plugins/authentication/aws_auth.py +14 -5
- eodag/plugins/authentication/base.py +10 -1
- eodag/plugins/authentication/generic.py +14 -3
- eodag/plugins/authentication/header.py +12 -4
- eodag/plugins/authentication/keycloak.py +41 -22
- eodag/plugins/authentication/oauth.py +11 -1
- eodag/plugins/authentication/openid_connect.py +178 -163
- eodag/plugins/authentication/qsauth.py +12 -4
- eodag/plugins/authentication/sas_auth.py +19 -2
- eodag/plugins/authentication/token.py +57 -10
- eodag/plugins/authentication/token_exchange.py +19 -19
- eodag/plugins/crunch/base.py +4 -1
- eodag/plugins/crunch/filter_date.py +5 -2
- eodag/plugins/crunch/filter_latest_intersect.py +5 -4
- eodag/plugins/crunch/filter_latest_tpl_name.py +1 -1
- eodag/plugins/crunch/filter_overlap.py +5 -7
- eodag/plugins/crunch/filter_property.py +4 -3
- eodag/plugins/download/aws.py +39 -22
- eodag/plugins/download/base.py +11 -11
- eodag/plugins/download/creodias_s3.py +11 -2
- eodag/plugins/download/http.py +86 -52
- eodag/plugins/download/s3rest.py +20 -18
- eodag/plugins/manager.py +168 -23
- eodag/plugins/search/base.py +33 -14
- eodag/plugins/search/build_search_result.py +55 -51
- eodag/plugins/search/cop_marine.py +112 -29
- eodag/plugins/search/creodias_s3.py +20 -5
- eodag/plugins/search/csw.py +41 -1
- eodag/plugins/search/data_request_search.py +109 -9
- eodag/plugins/search/qssearch.py +532 -152
- eodag/plugins/search/static_stac_search.py +20 -21
- eodag/resources/ext_product_types.json +1 -1
- eodag/resources/product_types.yml +187 -56
- eodag/resources/providers.yml +1610 -1701
- eodag/resources/stac.yml +3 -163
- eodag/resources/user_conf_template.yml +112 -97
- eodag/rest/config.py +1 -2
- eodag/rest/constants.py +0 -1
- eodag/rest/core.py +61 -51
- eodag/rest/errors.py +181 -0
- eodag/rest/server.py +24 -325
- eodag/rest/stac.py +93 -544
- eodag/rest/types/eodag_search.py +13 -8
- eodag/rest/types/queryables.py +1 -2
- eodag/rest/types/stac_search.py +11 -2
- eodag/types/__init__.py +15 -3
- eodag/types/download_args.py +1 -1
- eodag/types/queryables.py +1 -2
- eodag/types/search_args.py +3 -3
- eodag/utils/__init__.py +77 -57
- eodag/utils/exceptions.py +23 -9
- eodag/utils/logging.py +37 -77
- eodag/utils/requests.py +1 -3
- eodag/utils/stac_reader.py +1 -1
- {eodag-3.0.0b3.dist-info → eodag-3.0.1.dist-info}/METADATA +11 -12
- eodag-3.0.1.dist-info/RECORD +109 -0
- {eodag-3.0.0b3.dist-info → eodag-3.0.1.dist-info}/WHEEL +1 -1
- {eodag-3.0.0b3.dist-info → eodag-3.0.1.dist-info}/entry_points.txt +1 -0
- eodag/resources/constraints/climate-dt.json +0 -13
- eodag/resources/constraints/extremes-dt.json +0 -8
- eodag-3.0.0b3.dist-info/RECORD +0 -110
- {eodag-3.0.0b3.dist-info → eodag-3.0.1.dist-info}/LICENSE +0 -0
- {eodag-3.0.0b3.dist-info → eodag-3.0.1.dist-info}/top_level.txt +0 -0
eodag/api/core.py
CHANGED
|
@@ -17,13 +17,25 @@
|
|
|
17
17
|
# limitations under the License.
|
|
18
18
|
from __future__ import annotations
|
|
19
19
|
|
|
20
|
+
import datetime
|
|
20
21
|
import logging
|
|
21
22
|
import os
|
|
22
23
|
import re
|
|
23
24
|
import shutil
|
|
24
25
|
import tempfile
|
|
25
26
|
from operator import itemgetter
|
|
26
|
-
from typing import
|
|
27
|
+
from typing import (
|
|
28
|
+
TYPE_CHECKING,
|
|
29
|
+
Annotated,
|
|
30
|
+
Any,
|
|
31
|
+
Dict,
|
|
32
|
+
Iterator,
|
|
33
|
+
List,
|
|
34
|
+
Optional,
|
|
35
|
+
Set,
|
|
36
|
+
Tuple,
|
|
37
|
+
Union,
|
|
38
|
+
)
|
|
27
39
|
|
|
28
40
|
import geojson
|
|
29
41
|
import pkg_resources
|
|
@@ -35,11 +47,16 @@ from whoosh.fields import Schema
|
|
|
35
47
|
from whoosh.index import create_in, exists_in, open_dir
|
|
36
48
|
from whoosh.qparser import QueryParser
|
|
37
49
|
|
|
38
|
-
from eodag.api.product.metadata_mapping import
|
|
50
|
+
from eodag.api.product.metadata_mapping import (
|
|
51
|
+
ONLINE_STATUS,
|
|
52
|
+
mtd_cfg_as_conversion_and_querypath,
|
|
53
|
+
)
|
|
39
54
|
from eodag.api.search_result import SearchResult
|
|
40
55
|
from eodag.config import (
|
|
56
|
+
PLUGINS_TOPICS_KEYS,
|
|
41
57
|
PluginConfig,
|
|
42
58
|
SimpleYamlProxyConfig,
|
|
59
|
+
credentials_in_auth,
|
|
43
60
|
get_ext_product_types_conf,
|
|
44
61
|
load_default_config,
|
|
45
62
|
load_stac_provider_config,
|
|
@@ -48,10 +65,12 @@ from eodag.config import (
|
|
|
48
65
|
override_config_from_file,
|
|
49
66
|
override_config_from_mapping,
|
|
50
67
|
provider_config_init,
|
|
68
|
+
share_credentials,
|
|
51
69
|
)
|
|
52
70
|
from eodag.plugins.manager import PluginManager
|
|
53
71
|
from eodag.plugins.search import PreparedSearch
|
|
54
72
|
from eodag.plugins.search.build_search_result import BuildPostSearchResult
|
|
73
|
+
from eodag.plugins.search.qssearch import PostJsonSearch
|
|
55
74
|
from eodag.types import model_fields_to_annotated
|
|
56
75
|
from eodag.types.queryables import CommonQueryables
|
|
57
76
|
from eodag.types.whoosh import EODAGQueryParser
|
|
@@ -74,9 +93,7 @@ from eodag.utils import (
|
|
|
74
93
|
uri_to_path,
|
|
75
94
|
)
|
|
76
95
|
from eodag.utils.exceptions import (
|
|
77
|
-
AuthenticationError,
|
|
78
96
|
EodagError,
|
|
79
|
-
MisconfiguredError,
|
|
80
97
|
NoMatchingProductType,
|
|
81
98
|
PluginImplementationError,
|
|
82
99
|
RequestError,
|
|
@@ -96,7 +113,7 @@ if TYPE_CHECKING:
|
|
|
96
113
|
from eodag.plugins.search.base import Search
|
|
97
114
|
from eodag.types import ProviderSortables
|
|
98
115
|
from eodag.types.download_args import DownloadConf
|
|
99
|
-
from eodag.utils import
|
|
116
|
+
from eodag.utils import DownloadedCallback, ProgressCallback, Unpack
|
|
100
117
|
|
|
101
118
|
logger = logging.getLogger("eodag.core")
|
|
102
119
|
|
|
@@ -166,10 +183,15 @@ class EODataAccessGateway:
|
|
|
166
183
|
# Second level override: From environment variables
|
|
167
184
|
override_config_from_env(self.providers_config)
|
|
168
185
|
|
|
186
|
+
# share credentials between updated plugins confs
|
|
187
|
+
share_credentials(self.providers_config)
|
|
188
|
+
|
|
169
189
|
# init updated providers conf
|
|
170
|
-
stac_provider_config = load_stac_provider_config()
|
|
171
190
|
for provider in self.providers_config.keys():
|
|
172
|
-
provider_config_init(
|
|
191
|
+
provider_config_init(
|
|
192
|
+
self.providers_config[provider],
|
|
193
|
+
load_stac_provider_config(),
|
|
194
|
+
)
|
|
173
195
|
|
|
174
196
|
# re-build _plugins_manager using up-to-date providers_config
|
|
175
197
|
self._plugins_manager.rebuild(self.providers_config)
|
|
@@ -215,7 +237,6 @@ class EODataAccessGateway:
|
|
|
215
237
|
os.path.join(self.conf_dir, "shp"),
|
|
216
238
|
)
|
|
217
239
|
self.set_locations_conf(locations_conf_path)
|
|
218
|
-
self.search_errors: Set = set()
|
|
219
240
|
|
|
220
241
|
def get_version(self) -> str:
|
|
221
242
|
"""Get eodag package version"""
|
|
@@ -333,14 +354,24 @@ class EODataAccessGateway:
|
|
|
333
354
|
preferred, priority = max(providers_with_priority, key=itemgetter(1))
|
|
334
355
|
return preferred, priority
|
|
335
356
|
|
|
336
|
-
def update_providers_config(
|
|
357
|
+
def update_providers_config(
|
|
358
|
+
self,
|
|
359
|
+
yaml_conf: Optional[str] = None,
|
|
360
|
+
dict_conf: Optional[Dict[str, Any]] = None,
|
|
361
|
+
) -> None:
|
|
337
362
|
"""Update providers configuration with given input.
|
|
338
363
|
Can be used to add a provider to existing configuration or update
|
|
339
364
|
an existing one.
|
|
340
365
|
|
|
341
366
|
:param yaml_conf: YAML formated provider configuration
|
|
367
|
+
:param dict_conf: provider configuration as dictionary in place of ``yaml_conf``
|
|
342
368
|
"""
|
|
343
|
-
|
|
369
|
+
if dict_conf is not None:
|
|
370
|
+
conf_update = dict_conf
|
|
371
|
+
elif yaml_conf is not None:
|
|
372
|
+
conf_update = yaml.safe_load(yaml_conf)
|
|
373
|
+
else:
|
|
374
|
+
return None
|
|
344
375
|
|
|
345
376
|
# restore the pruned configuration
|
|
346
377
|
for provider in list(self._pruned_providers_config.keys()):
|
|
@@ -355,9 +386,14 @@ class EODataAccessGateway:
|
|
|
355
386
|
|
|
356
387
|
override_config_from_mapping(self.providers_config, conf_update)
|
|
357
388
|
|
|
358
|
-
|
|
389
|
+
# share credentials between updated plugins confs
|
|
390
|
+
share_credentials(self.providers_config)
|
|
391
|
+
|
|
359
392
|
for provider in conf_update.keys():
|
|
360
|
-
provider_config_init(
|
|
393
|
+
provider_config_init(
|
|
394
|
+
self.providers_config[provider],
|
|
395
|
+
load_stac_provider_config(),
|
|
396
|
+
)
|
|
361
397
|
setattr(self.providers_config[provider], "product_types_fetched", False)
|
|
362
398
|
# re-create _plugins_manager using up-to-date providers_config
|
|
363
399
|
self._plugins_manager.build_product_type_to_provider_config_map()
|
|
@@ -419,14 +455,11 @@ class EODataAccessGateway:
|
|
|
419
455
|
|
|
420
456
|
# api plugin usage: remove unneeded search/download/auth plugin conf
|
|
421
457
|
if conf_dict[name].get("api"):
|
|
422
|
-
|
|
423
|
-
|
|
424
|
-
|
|
458
|
+
for k in PLUGINS_TOPICS_KEYS:
|
|
459
|
+
if k != "api":
|
|
460
|
+
conf_dict[name].pop(k, None)
|
|
425
461
|
|
|
426
|
-
|
|
427
|
-
provider_config_init(self.providers_config[name], load_stac_provider_config())
|
|
428
|
-
setattr(self.providers_config[name], "product_types_fetched", False)
|
|
429
|
-
self._plugins_manager.build_product_type_to_provider_config_map()
|
|
462
|
+
self.update_providers_config(dict_conf=conf_dict)
|
|
430
463
|
|
|
431
464
|
if priority is None:
|
|
432
465
|
self.set_preferred_provider(name)
|
|
@@ -452,12 +485,7 @@ class EODataAccessGateway:
|
|
|
452
485
|
|
|
453
486
|
# check authentication
|
|
454
487
|
if hasattr(conf, "api") and getattr(conf.api, "need_auth", False):
|
|
455
|
-
credentials_exist =
|
|
456
|
-
[
|
|
457
|
-
cred is not None
|
|
458
|
-
for cred in getattr(conf.api, "credentials", {}).values()
|
|
459
|
-
]
|
|
460
|
-
)
|
|
488
|
+
credentials_exist = credentials_in_auth(conf.api)
|
|
461
489
|
if not credentials_exist:
|
|
462
490
|
# credentials needed but not found
|
|
463
491
|
self._pruned_providers_config[provider] = self.providers_config.pop(
|
|
@@ -469,7 +497,7 @@ class EODataAccessGateway:
|
|
|
469
497
|
provider,
|
|
470
498
|
)
|
|
471
499
|
elif hasattr(conf, "search") and getattr(conf.search, "need_auth", False):
|
|
472
|
-
if not hasattr(conf, "auth"):
|
|
500
|
+
if not hasattr(conf, "auth") and not hasattr(conf, "search_auth"):
|
|
473
501
|
# credentials needed but no auth plugin was found
|
|
474
502
|
self._pruned_providers_config[provider] = self.providers_config.pop(
|
|
475
503
|
provider
|
|
@@ -480,11 +508,13 @@ class EODataAccessGateway:
|
|
|
480
508
|
provider,
|
|
481
509
|
)
|
|
482
510
|
continue
|
|
483
|
-
credentials_exist =
|
|
484
|
-
|
|
485
|
-
|
|
486
|
-
|
|
487
|
-
|
|
511
|
+
credentials_exist = (
|
|
512
|
+
hasattr(conf, "search_auth")
|
|
513
|
+
and credentials_in_auth(conf.search_auth)
|
|
514
|
+
) or (
|
|
515
|
+
not hasattr(conf, "search_auth")
|
|
516
|
+
and hasattr(conf, "auth")
|
|
517
|
+
and credentials_in_auth(conf.auth)
|
|
488
518
|
)
|
|
489
519
|
if not credentials_exist:
|
|
490
520
|
# credentials needed but not found
|
|
@@ -599,21 +629,32 @@ class EODataAccessGateway:
|
|
|
599
629
|
def fetch_product_types_list(self, provider: Optional[str] = None) -> None:
|
|
600
630
|
"""Fetch product types list and update if needed
|
|
601
631
|
|
|
602
|
-
:param provider:
|
|
603
|
-
should be updated. Defaults to all providers (None value).
|
|
632
|
+
:param provider: The name of a provider or provider-group for which product types
|
|
633
|
+
list should be updated. Defaults to all providers (None value).
|
|
604
634
|
"""
|
|
635
|
+
providers_to_fetch = list(self.providers_config.keys())
|
|
636
|
+
# check if some providers are grouped under a group name which is not a provider name
|
|
605
637
|
if provider is not None and provider not in self.providers_config:
|
|
606
|
-
|
|
638
|
+
providers_to_fetch = [
|
|
639
|
+
p
|
|
640
|
+
for p, pconf in self.providers_config.items()
|
|
641
|
+
if provider == getattr(pconf, "group", None)
|
|
642
|
+
]
|
|
643
|
+
if providers_to_fetch:
|
|
644
|
+
logger.info(
|
|
645
|
+
f"Fetch product types for {provider} group: {', '.join(providers_to_fetch)}"
|
|
646
|
+
)
|
|
647
|
+
else:
|
|
648
|
+
return None
|
|
649
|
+
elif provider is not None:
|
|
650
|
+
providers_to_fetch = [provider]
|
|
607
651
|
|
|
608
652
|
# providers discovery confs that are fetchable
|
|
609
653
|
providers_discovery_configs_fetchable: Dict[str, Any] = {}
|
|
610
654
|
# check if any provider has not already been fetched for product types
|
|
611
655
|
already_fetched = True
|
|
612
|
-
for provider_to_fetch
|
|
613
|
-
|
|
614
|
-
if provider
|
|
615
|
-
else self.providers_config.items()
|
|
616
|
-
):
|
|
656
|
+
for provider_to_fetch in providers_to_fetch:
|
|
657
|
+
provider_config = self.providers_config[provider_to_fetch]
|
|
617
658
|
# get discovery conf
|
|
618
659
|
if hasattr(provider_config, "search"):
|
|
619
660
|
provider_search_config = provider_config.search
|
|
@@ -735,11 +776,20 @@ class EODataAccessGateway:
|
|
|
735
776
|
) -> Optional[Dict[str, Any]]:
|
|
736
777
|
"""Fetch providers for product types
|
|
737
778
|
|
|
738
|
-
:param provider:
|
|
739
|
-
providers (None value).
|
|
779
|
+
:param provider: The name of a provider or provider-group to fetch. Defaults to
|
|
780
|
+
all providers (None value).
|
|
740
781
|
:returns: external product types configuration
|
|
741
782
|
"""
|
|
742
|
-
|
|
783
|
+
grouped_providers = [
|
|
784
|
+
p
|
|
785
|
+
for p, provider_config in self.providers_config.items()
|
|
786
|
+
if provider == getattr(provider_config, "group", None)
|
|
787
|
+
]
|
|
788
|
+
if provider and provider not in self.providers_config and grouped_providers:
|
|
789
|
+
logger.info(
|
|
790
|
+
f"Discover product types for {provider} group: {', '.join(grouped_providers)}"
|
|
791
|
+
)
|
|
792
|
+
elif provider and provider not in self.providers_config:
|
|
743
793
|
raise UnsupportedProvider(
|
|
744
794
|
f"The requested provider is not (yet) supported: {provider}"
|
|
745
795
|
)
|
|
@@ -748,7 +798,9 @@ class EODataAccessGateway:
|
|
|
748
798
|
p
|
|
749
799
|
for p in (
|
|
750
800
|
[
|
|
751
|
-
|
|
801
|
+
p
|
|
802
|
+
for p in self.providers_config
|
|
803
|
+
if p in grouped_providers + [provider]
|
|
752
804
|
]
|
|
753
805
|
if provider
|
|
754
806
|
else self.available_providers()
|
|
@@ -762,7 +814,9 @@ class EODataAccessGateway:
|
|
|
762
814
|
search_plugin_config = self.providers_config[provider].api
|
|
763
815
|
else:
|
|
764
816
|
return None
|
|
765
|
-
if getattr(search_plugin_config, "discover_product_types",
|
|
817
|
+
if getattr(search_plugin_config, "discover_product_types", {}).get(
|
|
818
|
+
"fetch_url", None
|
|
819
|
+
):
|
|
766
820
|
search_plugin: Union[Search, Api] = next(
|
|
767
821
|
self._plugins_manager.get_search_plugins(provider=provider)
|
|
768
822
|
)
|
|
@@ -773,23 +827,15 @@ class EODataAccessGateway:
|
|
|
773
827
|
continue
|
|
774
828
|
# append auth to search plugin if needed
|
|
775
829
|
if getattr(search_plugin.config, "need_auth", False):
|
|
776
|
-
|
|
777
|
-
search_plugin.provider
|
|
778
|
-
|
|
779
|
-
|
|
780
|
-
getattr(auth_plugin, "authenticate", None)
|
|
830
|
+
if auth := self._plugins_manager.get_auth(
|
|
831
|
+
search_plugin.provider,
|
|
832
|
+
getattr(search_plugin.config, "api_endpoint", None),
|
|
833
|
+
search_plugin.config,
|
|
781
834
|
):
|
|
782
|
-
|
|
783
|
-
kwargs["auth"] = auth_plugin.authenticate()
|
|
784
|
-
except (AuthenticationError, MisconfiguredError) as e:
|
|
785
|
-
logger.warning(
|
|
786
|
-
f"Could not authenticate on {provider}: {str(e)}"
|
|
787
|
-
)
|
|
788
|
-
ext_product_types_conf[provider] = None
|
|
789
|
-
continue
|
|
835
|
+
kwargs["auth"] = auth
|
|
790
836
|
else:
|
|
791
|
-
logger.
|
|
792
|
-
f"Could not authenticate on {provider}
|
|
837
|
+
logger.debug(
|
|
838
|
+
f"Could not authenticate on {provider} for product types discovery"
|
|
793
839
|
)
|
|
794
840
|
ext_product_types_conf[provider] = None
|
|
795
841
|
continue
|
|
@@ -815,7 +861,9 @@ class EODataAccessGateway:
|
|
|
815
861
|
) or getattr(self.providers_config[provider], "api", None)
|
|
816
862
|
if search_plugin_config is None:
|
|
817
863
|
continue
|
|
818
|
-
if not
|
|
864
|
+
if not getattr(
|
|
865
|
+
search_plugin_config, "discover_product_types", {}
|
|
866
|
+
).get("fetch_url", None):
|
|
819
867
|
# conf has been updated and provider product types are no more discoverable
|
|
820
868
|
continue
|
|
821
869
|
provider_products_config = (
|
|
@@ -874,7 +922,7 @@ class EODataAccessGateway:
|
|
|
874
922
|
new_product_types.append(new_product_type)
|
|
875
923
|
if new_product_types:
|
|
876
924
|
logger.debug(
|
|
877
|
-
f"Added
|
|
925
|
+
f"Added {len(new_product_types)} product types for {provider}"
|
|
878
926
|
)
|
|
879
927
|
|
|
880
928
|
elif provider not in self.providers_config:
|
|
@@ -1046,20 +1094,28 @@ class EODataAccessGateway:
|
|
|
1046
1094
|
|
|
1047
1095
|
# datetime filtering
|
|
1048
1096
|
if missionStartDate or missionEndDate:
|
|
1097
|
+
min_aware = datetime.datetime.min.replace(tzinfo=datetime.timezone.utc)
|
|
1098
|
+
max_aware = datetime.datetime.max.replace(tzinfo=datetime.timezone.utc)
|
|
1049
1099
|
guesses = [
|
|
1050
1100
|
g
|
|
1051
1101
|
for g in guesses
|
|
1052
1102
|
if (
|
|
1053
|
-
|
|
1054
|
-
|
|
1055
|
-
|
|
1056
|
-
|
|
1057
|
-
|
|
1058
|
-
|
|
1059
|
-
|
|
1060
|
-
|
|
1061
|
-
|
|
1062
|
-
|
|
1103
|
+
max(
|
|
1104
|
+
rfc3339_str_to_datetime(missionStartDate)
|
|
1105
|
+
if missionStartDate
|
|
1106
|
+
else min_aware,
|
|
1107
|
+
rfc3339_str_to_datetime(g["missionStartDate"])
|
|
1108
|
+
if g.get("missionStartDate")
|
|
1109
|
+
else min_aware,
|
|
1110
|
+
)
|
|
1111
|
+
<= min(
|
|
1112
|
+
rfc3339_str_to_datetime(missionEndDate)
|
|
1113
|
+
if missionEndDate
|
|
1114
|
+
else max_aware,
|
|
1115
|
+
rfc3339_str_to_datetime(g["missionEndDate"])
|
|
1116
|
+
if g.get("missionEndDate")
|
|
1117
|
+
else max_aware,
|
|
1118
|
+
)
|
|
1063
1119
|
)
|
|
1064
1120
|
]
|
|
1065
1121
|
|
|
@@ -1155,7 +1211,7 @@ class EODataAccessGateway:
|
|
|
1155
1211
|
items_per_page=items_per_page,
|
|
1156
1212
|
)
|
|
1157
1213
|
|
|
1158
|
-
|
|
1214
|
+
errors: List[Tuple[str, Exception]] = []
|
|
1159
1215
|
# Loop over available providers and return the first non-empty results
|
|
1160
1216
|
for i, search_plugin in enumerate(search_plugins):
|
|
1161
1217
|
search_plugin.clear()
|
|
@@ -1165,17 +1221,19 @@ class EODataAccessGateway:
|
|
|
1165
1221
|
raise_errors=raise_errors,
|
|
1166
1222
|
**search_kwargs,
|
|
1167
1223
|
)
|
|
1224
|
+
errors.extend(search_results.errors)
|
|
1168
1225
|
if len(search_results) == 0 and i < len(search_plugins) - 1:
|
|
1169
1226
|
logger.warning(
|
|
1170
1227
|
f"No result could be obtained from provider {search_plugin.provider}, "
|
|
1171
1228
|
"we will try to get the data from another provider",
|
|
1172
1229
|
)
|
|
1173
1230
|
elif len(search_results) > 0:
|
|
1231
|
+
search_results.errors = errors
|
|
1174
1232
|
return search_results
|
|
1175
1233
|
|
|
1176
1234
|
if i > 1:
|
|
1177
1235
|
logger.error("No result could be obtained from any available provider")
|
|
1178
|
-
return SearchResult([], 0) if count else SearchResult([])
|
|
1236
|
+
return SearchResult([], 0, errors) if count else SearchResult([], errors=errors)
|
|
1179
1237
|
|
|
1180
1238
|
def search_iter_page(
|
|
1181
1239
|
self,
|
|
@@ -1435,7 +1493,7 @@ class EODataAccessGateway:
|
|
|
1435
1493
|
)
|
|
1436
1494
|
or DEFAULT_MAX_ITEMS_PER_PAGE
|
|
1437
1495
|
)
|
|
1438
|
-
logger.
|
|
1496
|
+
logger.info(
|
|
1439
1497
|
"Searching for all the products with provider %s and a maximum of %s "
|
|
1440
1498
|
"items per page.",
|
|
1441
1499
|
search_plugin.provider,
|
|
@@ -1503,7 +1561,7 @@ class EODataAccessGateway:
|
|
|
1503
1561
|
try:
|
|
1504
1562
|
product_type = self.get_product_type_from_alias(product_type)
|
|
1505
1563
|
except NoMatchingProductType:
|
|
1506
|
-
logger.
|
|
1564
|
+
logger.debug("product type %s not found", product_type)
|
|
1507
1565
|
get_search_plugins_kwargs = dict(provider=provider, product_type=product_type)
|
|
1508
1566
|
search_plugins = self._plugins_manager.get_search_plugins(
|
|
1509
1567
|
**get_search_plugins_kwargs
|
|
@@ -1526,7 +1584,7 @@ class EODataAccessGateway:
|
|
|
1526
1584
|
"max_items_per_page", DEFAULT_MAX_ITEMS_PER_PAGE
|
|
1527
1585
|
)
|
|
1528
1586
|
kwargs.update(items_per_page=items_per_page)
|
|
1529
|
-
if isinstance(plugin,
|
|
1587
|
+
if isinstance(plugin, PostJsonSearch):
|
|
1530
1588
|
kwargs.update(
|
|
1531
1589
|
items_per_page=items_per_page,
|
|
1532
1590
|
_dc_qs=_dc_qs,
|
|
@@ -1544,9 +1602,10 @@ class EODataAccessGateway:
|
|
|
1544
1602
|
**kwargs,
|
|
1545
1603
|
):
|
|
1546
1604
|
results.data.extend(page_results.data)
|
|
1547
|
-
except Exception:
|
|
1605
|
+
except Exception as e:
|
|
1548
1606
|
if kwargs.get("raise_errors"):
|
|
1549
1607
|
raise
|
|
1608
|
+
logger.warning(e)
|
|
1550
1609
|
continue
|
|
1551
1610
|
|
|
1552
1611
|
# try using crunch to get unique result
|
|
@@ -1584,16 +1643,12 @@ class EODataAccessGateway:
|
|
|
1584
1643
|
|
|
1585
1644
|
# append auth if needed
|
|
1586
1645
|
if getattr(plugin.config, "need_auth", False):
|
|
1587
|
-
|
|
1588
|
-
|
|
1589
|
-
|
|
1590
|
-
|
|
1591
|
-
|
|
1592
|
-
|
|
1593
|
-
else:
|
|
1594
|
-
logger.warning(
|
|
1595
|
-
f"Could not authenticate on {provider} using {auth_plugin} plugin"
|
|
1596
|
-
)
|
|
1646
|
+
if auth := self._plugins_manager.get_auth(
|
|
1647
|
+
plugin.provider,
|
|
1648
|
+
getattr(plugin.config, "api_endpoint", None),
|
|
1649
|
+
plugin.config,
|
|
1650
|
+
):
|
|
1651
|
+
kwargs["auth"] = auth
|
|
1597
1652
|
|
|
1598
1653
|
product_type_config = plugin.discover_product_types(**kwargs)
|
|
1599
1654
|
self.update_product_types_list({provider: product_type_config})
|
|
@@ -1732,12 +1787,10 @@ class EODataAccessGateway:
|
|
|
1732
1787
|
provider = preferred_provider
|
|
1733
1788
|
providers = [plugin.provider for plugin in search_plugins]
|
|
1734
1789
|
if provider not in providers:
|
|
1735
|
-
logger.
|
|
1736
|
-
"Product type '%s' is not available with provider '%s'.
|
|
1737
|
-
"Searching it on provider '%s' instead.",
|
|
1790
|
+
logger.debug(
|
|
1791
|
+
"Product type '%s' is not available with preferred provider '%s'.",
|
|
1738
1792
|
product_type,
|
|
1739
1793
|
provider,
|
|
1740
|
-
search_plugins[0].provider,
|
|
1741
1794
|
)
|
|
1742
1795
|
else:
|
|
1743
1796
|
provider_plugin = list(
|
|
@@ -1745,11 +1798,6 @@ class EODataAccessGateway:
|
|
|
1745
1798
|
)[0]
|
|
1746
1799
|
search_plugins.remove(provider_plugin)
|
|
1747
1800
|
search_plugins.insert(0, provider_plugin)
|
|
1748
|
-
logger.info(
|
|
1749
|
-
"Searching product type '%s' on provider: %s",
|
|
1750
|
-
product_type,
|
|
1751
|
-
search_plugins[0].provider,
|
|
1752
|
-
)
|
|
1753
1801
|
# Add product_types_config to plugin config. This dict contains product
|
|
1754
1802
|
# type metadata that will also be stored in each product's properties.
|
|
1755
1803
|
for search_plugin in search_plugins:
|
|
@@ -1793,6 +1841,7 @@ class EODataAccessGateway:
|
|
|
1793
1841
|
:param kwargs: Some other criteria that will be used to do the search
|
|
1794
1842
|
:returns: A collection of EO products matching the criteria
|
|
1795
1843
|
"""
|
|
1844
|
+
logger.info("Searching on provider %s", search_plugin.provider)
|
|
1796
1845
|
max_items_per_page = getattr(search_plugin.config, "pagination", {}).get(
|
|
1797
1846
|
"max_items_per_page", DEFAULT_MAX_ITEMS_PER_PAGE
|
|
1798
1847
|
)
|
|
@@ -1810,19 +1859,23 @@ class EODataAccessGateway:
|
|
|
1810
1859
|
max_items_per_page,
|
|
1811
1860
|
)
|
|
1812
1861
|
|
|
1813
|
-
need_auth = getattr(search_plugin.config, "need_auth", False)
|
|
1814
|
-
auth_plugin = self._plugins_manager.get_auth_plugin(search_plugin.provider)
|
|
1815
|
-
can_authenticate = callable(getattr(auth_plugin, "authenticate", None))
|
|
1816
|
-
|
|
1817
1862
|
results: List[EOProduct] = []
|
|
1818
1863
|
total_results: Optional[int] = 0 if count else None
|
|
1819
1864
|
|
|
1865
|
+
errors: List[Tuple[str, Exception]] = []
|
|
1866
|
+
|
|
1820
1867
|
try:
|
|
1821
1868
|
prep = PreparedSearch(count=count)
|
|
1822
|
-
if need_auth and auth_plugin and can_authenticate:
|
|
1823
|
-
prep.auth = auth_plugin.authenticate()
|
|
1824
1869
|
|
|
1825
|
-
|
|
1870
|
+
# append auth if needed
|
|
1871
|
+
if getattr(search_plugin.config, "need_auth", False):
|
|
1872
|
+
if auth := self._plugins_manager.get_auth(
|
|
1873
|
+
search_plugin.provider,
|
|
1874
|
+
getattr(search_plugin.config, "api_endpoint", None),
|
|
1875
|
+
search_plugin.config,
|
|
1876
|
+
):
|
|
1877
|
+
prep.auth = auth
|
|
1878
|
+
|
|
1826
1879
|
prep.page = kwargs.pop("page", None)
|
|
1827
1880
|
prep.items_per_page = kwargs.pop("items_per_page", None)
|
|
1828
1881
|
|
|
@@ -1876,12 +1929,31 @@ class EODataAccessGateway:
|
|
|
1876
1929
|
eo_product.product_type
|
|
1877
1930
|
)
|
|
1878
1931
|
except NoMatchingProductType:
|
|
1879
|
-
logger.
|
|
1932
|
+
logger.debug("product type %s not found", eo_product.product_type)
|
|
1880
1933
|
|
|
1881
1934
|
if eo_product.search_intersection is not None:
|
|
1882
1935
|
download_plugin = self._plugins_manager.get_download_plugin(
|
|
1883
1936
|
eo_product
|
|
1884
1937
|
)
|
|
1938
|
+
if len(eo_product.assets) > 0:
|
|
1939
|
+
matching_url = next(iter(eo_product.assets.values()))["href"]
|
|
1940
|
+
elif eo_product.properties.get("storageStatus") != ONLINE_STATUS:
|
|
1941
|
+
matching_url = eo_product.properties.get(
|
|
1942
|
+
"orderLink"
|
|
1943
|
+
) or eo_product.properties.get("downloadLink")
|
|
1944
|
+
else:
|
|
1945
|
+
matching_url = eo_product.properties.get("downloadLink")
|
|
1946
|
+
|
|
1947
|
+
try:
|
|
1948
|
+
auth_plugin = next(
|
|
1949
|
+
self._plugins_manager.get_auth_plugins(
|
|
1950
|
+
search_plugin.provider,
|
|
1951
|
+
matching_url=matching_url,
|
|
1952
|
+
matching_conf=download_plugin.config,
|
|
1953
|
+
)
|
|
1954
|
+
)
|
|
1955
|
+
except StopIteration:
|
|
1956
|
+
auth_plugin = None
|
|
1885
1957
|
eo_product.register_downloader(download_plugin, auth_plugin)
|
|
1886
1958
|
|
|
1887
1959
|
results.extend(res)
|
|
@@ -1911,13 +1983,6 @@ class EODataAccessGateway:
|
|
|
1911
1983
|
"the total number of products matching the search criteria"
|
|
1912
1984
|
)
|
|
1913
1985
|
except Exception as e:
|
|
1914
|
-
log_msg = f"No result from provider '{search_plugin.provider}' due to an error during search."
|
|
1915
|
-
if not raise_errors:
|
|
1916
|
-
log_msg += " Raise verbosity of log messages for details"
|
|
1917
|
-
logger.info(log_msg)
|
|
1918
|
-
# keep only the message from exception args
|
|
1919
|
-
if len(e.args) > 1:
|
|
1920
|
-
e.args = (e.args[0],)
|
|
1921
1986
|
if raise_errors:
|
|
1922
1987
|
# Raise the error, letting the application wrapping eodag know that
|
|
1923
1988
|
# something went bad. This way it will be able to decide what to do next
|
|
@@ -1927,8 +1992,8 @@ class EODataAccessGateway:
|
|
|
1927
1992
|
"Error while searching on provider %s (ignored):",
|
|
1928
1993
|
search_plugin.provider,
|
|
1929
1994
|
)
|
|
1930
|
-
|
|
1931
|
-
return SearchResult(results, total_results)
|
|
1995
|
+
errors.append((search_plugin.provider, e))
|
|
1996
|
+
return SearchResult(results, total_results, errors)
|
|
1932
1997
|
|
|
1933
1998
|
def crunch(self, results: SearchResult, **kwargs: Any) -> SearchResult:
|
|
1934
1999
|
"""Apply the filters given through the keyword arguments to the results
|
|
@@ -1982,7 +2047,7 @@ class EODataAccessGateway:
|
|
|
1982
2047
|
:param search_result: A collection of EO products resulting from a search
|
|
1983
2048
|
:param downloaded_callback: (optional) A method or a callable object which takes
|
|
1984
2049
|
as parameter the ``product``. You can use the base class
|
|
1985
|
-
:class:`~eodag.
|
|
2050
|
+
:class:`~eodag.utils.DownloadedCallback` and override
|
|
1986
2051
|
its ``__call__`` method. Will be called each time a product
|
|
1987
2052
|
finishes downloading
|
|
1988
2053
|
:param progress_callback: (optional) A method or a callable object
|
|
@@ -2061,12 +2126,12 @@ class EODataAccessGateway:
|
|
|
2061
2126
|
products = self.deserialize(filename)
|
|
2062
2127
|
for i, product in enumerate(products):
|
|
2063
2128
|
if product.downloader is None:
|
|
2129
|
+
downloader = self._plugins_manager.get_download_plugin(product)
|
|
2064
2130
|
auth = product.downloader_auth
|
|
2065
2131
|
if auth is None:
|
|
2066
|
-
auth = self._plugins_manager.get_auth_plugin(product
|
|
2067
|
-
products[i].register_downloader(
|
|
2068
|
-
|
|
2069
|
-
)
|
|
2132
|
+
auth = self._plugins_manager.get_auth_plugin(downloader, product)
|
|
2133
|
+
products[i].register_downloader(downloader, auth)
|
|
2134
|
+
|
|
2070
2135
|
return products
|
|
2071
2136
|
|
|
2072
2137
|
@_deprecated(
|
|
@@ -2090,7 +2155,7 @@ class EODataAccessGateway:
|
|
|
2090
2155
|
|
|
2091
2156
|
:param filename: A filename containing features encoded as a geojson
|
|
2092
2157
|
:param recursive: (optional) Browse recursively in child nodes if True
|
|
2093
|
-
:param max_connections: (optional) Maximum number of connections for HTTP requests
|
|
2158
|
+
:param max_connections: (optional) Maximum number of connections for concurrent HTTP requests
|
|
2094
2159
|
:param provider: (optional) Data provider
|
|
2095
2160
|
:param productType: (optional) Data product type
|
|
2096
2161
|
:param timeout: (optional) Timeout in seconds for each internal HTTP request
|
|
@@ -2197,12 +2262,11 @@ class EODataAccessGateway:
|
|
|
2197
2262
|
|
|
2198
2263
|
def _setup_downloader(self, product: EOProduct) -> None:
|
|
2199
2264
|
if product.downloader is None:
|
|
2265
|
+
downloader = self._plugins_manager.get_download_plugin(product)
|
|
2200
2266
|
auth = product.downloader_auth
|
|
2201
2267
|
if auth is None:
|
|
2202
|
-
auth = self._plugins_manager.get_auth_plugin(product
|
|
2203
|
-
product.register_downloader(
|
|
2204
|
-
self._plugins_manager.get_download_plugin(product), auth
|
|
2205
|
-
)
|
|
2268
|
+
auth = self._plugins_manager.get_auth_plugin(downloader, product)
|
|
2269
|
+
product.register_downloader(downloader, auth)
|
|
2206
2270
|
|
|
2207
2271
|
def get_cruncher(self, name: str, **options: Any) -> Crunch:
|
|
2208
2272
|
"""Build a crunch plugin from a configuration
|
|
@@ -2254,7 +2318,7 @@ class EODataAccessGateway:
|
|
|
2254
2318
|
|
|
2255
2319
|
for plugin in self._plugins_manager.get_search_plugins(product_type, provider):
|
|
2256
2320
|
if getattr(plugin.config, "need_auth", False) and (
|
|
2257
|
-
auth := self._plugins_manager.get_auth_plugin(plugin
|
|
2321
|
+
auth := self._plugins_manager.get_auth_plugin(plugin)
|
|
2258
2322
|
):
|
|
2259
2323
|
plugin.auth = auth.authenticate()
|
|
2260
2324
|
providers_queryables[plugin.provider] = plugin.list_queryables(
|
|
@@ -2284,7 +2348,7 @@ class EODataAccessGateway:
|
|
|
2284
2348
|
"""For each provider, gives its available sortable parameter(s) and its maximum
|
|
2285
2349
|
number of them if it supports the sorting feature, otherwise gives None.
|
|
2286
2350
|
|
|
2287
|
-
:returns: A
|
|
2351
|
+
:returns: A dictionary with providers as keys and dictionary of sortable parameter(s) and
|
|
2288
2352
|
its (their) maximum number as value(s).
|
|
2289
2353
|
:raises: :class:`~eodag.utils.exceptions.UnsupportedProvider`
|
|
2290
2354
|
"""
|