eodag 4.0.0a2__py3-none-any.whl → 4.0.0a4__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- eodag/api/collection.py +10 -9
- eodag/api/core.py +233 -335
- eodag/api/product/_product.py +3 -3
- eodag/api/provider.py +990 -0
- eodag/cli.py +1 -3
- eodag/config.py +73 -444
- eodag/plugins/authentication/token.py +0 -1
- eodag/plugins/download/http.py +0 -1
- eodag/plugins/manager.py +24 -34
- eodag/resources/ext_collections.json +1 -1
- eodag/resources/ext_product_types.json +1 -1
- eodag/resources/providers.yml +4 -4
- {eodag-4.0.0a2.dist-info → eodag-4.0.0a4.dist-info}/METADATA +1 -1
- {eodag-4.0.0a2.dist-info → eodag-4.0.0a4.dist-info}/RECORD +18 -17
- {eodag-4.0.0a2.dist-info → eodag-4.0.0a4.dist-info}/WHEEL +0 -0
- {eodag-4.0.0a2.dist-info → eodag-4.0.0a4.dist-info}/entry_points.txt +0 -0
- {eodag-4.0.0a2.dist-info → eodag-4.0.0a4.dist-info}/licenses/LICENSE +0 -0
- {eodag-4.0.0a2.dist-info → eodag-4.0.0a4.dist-info}/top_level.txt +0 -0
eodag/cli.py
CHANGED
|
@@ -484,9 +484,7 @@ def list_col(ctx: Context, **kwargs: Any) -> None:
|
|
|
484
484
|
click.echo(text_wrapper.fill(str(value)))
|
|
485
485
|
except UnsupportedProvider:
|
|
486
486
|
click.echo("Unsupported provider. You may have a typo")
|
|
487
|
-
click.echo(
|
|
488
|
-
"Available providers: {}".format(", ".join(dag.available_providers()))
|
|
489
|
-
)
|
|
487
|
+
click.echo("Available providers: {}".format(", ".join(dag.providers.names)))
|
|
490
488
|
sys.exit(1)
|
|
491
489
|
|
|
492
490
|
|
eodag/config.py
CHANGED
|
@@ -19,21 +19,8 @@ from __future__ import annotations
|
|
|
19
19
|
|
|
20
20
|
import logging
|
|
21
21
|
import os
|
|
22
|
-
import tempfile
|
|
23
22
|
from importlib.resources import files as res_files
|
|
24
|
-
from
|
|
25
|
-
from typing import (
|
|
26
|
-
Annotated,
|
|
27
|
-
Any,
|
|
28
|
-
ItemsView,
|
|
29
|
-
Iterator,
|
|
30
|
-
Literal,
|
|
31
|
-
Optional,
|
|
32
|
-
TypedDict,
|
|
33
|
-
Union,
|
|
34
|
-
ValuesView,
|
|
35
|
-
get_type_hints,
|
|
36
|
-
)
|
|
23
|
+
from typing import TYPE_CHECKING, Annotated, Any, Literal, Optional, TypedDict, Union
|
|
37
24
|
|
|
38
25
|
import orjson
|
|
39
26
|
import requests
|
|
@@ -42,25 +29,27 @@ import yaml.parser
|
|
|
42
29
|
from annotated_types import Gt
|
|
43
30
|
from jsonpath_ng import JSONPath
|
|
44
31
|
|
|
45
|
-
from eodag.api.product.metadata_mapping import mtd_cfg_as_conversion_and_querypath
|
|
46
32
|
from eodag.utils import (
|
|
47
33
|
HTTP_REQ_TIMEOUT,
|
|
48
|
-
STAC_SEARCH_PLUGINS,
|
|
49
34
|
USER_AGENT,
|
|
50
35
|
cached_yaml_load,
|
|
51
36
|
cached_yaml_load_all,
|
|
52
|
-
cast_scalar_value,
|
|
53
37
|
deepcopy,
|
|
54
38
|
dict_items_recursive_apply,
|
|
55
39
|
merge_mappings,
|
|
56
|
-
slugify,
|
|
57
40
|
sort_dict,
|
|
58
41
|
string_to_jsonpath,
|
|
59
|
-
update_nested_dict,
|
|
60
42
|
uri_to_path,
|
|
61
43
|
)
|
|
62
44
|
from eodag.utils.exceptions import ValidationError
|
|
63
45
|
|
|
46
|
+
if TYPE_CHECKING:
|
|
47
|
+
from typing import ItemsView, Iterator, ValuesView
|
|
48
|
+
|
|
49
|
+
from typing_extensions import Self
|
|
50
|
+
|
|
51
|
+
from eodag.api.provider import ProviderConfig
|
|
52
|
+
|
|
64
53
|
logger = logging.getLogger("eodag.config")
|
|
65
54
|
|
|
66
55
|
EXT_COLLECTIONS_CONF_URI = (
|
|
@@ -105,115 +94,6 @@ class SimpleYamlProxyConfig:
|
|
|
105
94
|
self.source.update(other.source)
|
|
106
95
|
|
|
107
96
|
|
|
108
|
-
class ProviderConfig(yaml.YAMLObject):
|
|
109
|
-
"""Representation of eodag configuration.
|
|
110
|
-
|
|
111
|
-
:param name: The name of the provider
|
|
112
|
-
:param priority: (optional) The priority of the provider while searching a product.
|
|
113
|
-
Lower value means lower priority. (Default: 0)
|
|
114
|
-
:param api: (optional) The configuration of a plugin of type Api
|
|
115
|
-
:param search: (optional) The configuration of a plugin of type Search
|
|
116
|
-
:param products: (optional) The collections supported by the provider
|
|
117
|
-
:param download: (optional) The configuration of a plugin of type Download
|
|
118
|
-
:param auth: (optional) The configuration of a plugin of type Authentication
|
|
119
|
-
:param search_auth: (optional) The configuration of a plugin of type Authentication for search
|
|
120
|
-
:param download_auth: (optional) The configuration of a plugin of type Authentication for download
|
|
121
|
-
:param kwargs: Additional configuration variables for this provider
|
|
122
|
-
"""
|
|
123
|
-
|
|
124
|
-
name: str
|
|
125
|
-
group: str
|
|
126
|
-
priority: int = 0 # Set default priority to 0
|
|
127
|
-
roles: list[str]
|
|
128
|
-
description: str
|
|
129
|
-
url: str
|
|
130
|
-
api: PluginConfig
|
|
131
|
-
search: PluginConfig
|
|
132
|
-
products: dict[str, Any]
|
|
133
|
-
download: PluginConfig
|
|
134
|
-
auth: PluginConfig
|
|
135
|
-
search_auth: PluginConfig
|
|
136
|
-
download_auth: PluginConfig
|
|
137
|
-
collections_fetched: bool # set in core.update_collections_list
|
|
138
|
-
|
|
139
|
-
yaml_loader = yaml.Loader
|
|
140
|
-
yaml_dumper = yaml.SafeDumper
|
|
141
|
-
yaml_tag = "!provider"
|
|
142
|
-
|
|
143
|
-
@classmethod
|
|
144
|
-
def from_yaml(cls, loader: yaml.Loader, node: Any) -> Iterator[ProviderConfig]:
|
|
145
|
-
"""Build a :class:`~eodag.config.ProviderConfig` from Yaml"""
|
|
146
|
-
cls.validate(tuple(node_key.value for node_key, _ in node.value))
|
|
147
|
-
for node_key, node_value in node.value:
|
|
148
|
-
if node_key.value == "name":
|
|
149
|
-
node_value.value = slugify(node_value.value).replace("-", "_")
|
|
150
|
-
break
|
|
151
|
-
return loader.construct_yaml_object(node, cls)
|
|
152
|
-
|
|
153
|
-
@classmethod
|
|
154
|
-
def from_mapping(cls, mapping: dict[str, Any]) -> ProviderConfig:
|
|
155
|
-
"""Build a :class:`~eodag.config.ProviderConfig` from a mapping"""
|
|
156
|
-
cls.validate(mapping)
|
|
157
|
-
for key in PLUGINS_TOPICS_KEYS:
|
|
158
|
-
if key in mapping:
|
|
159
|
-
mapping[key] = PluginConfig.from_mapping(mapping[key])
|
|
160
|
-
c = cls()
|
|
161
|
-
c.__dict__.update(mapping)
|
|
162
|
-
return c
|
|
163
|
-
|
|
164
|
-
@staticmethod
|
|
165
|
-
def validate(config_keys: Union[tuple[str, ...], dict[str, Any]]) -> None:
|
|
166
|
-
"""Validate a :class:`~eodag.config.ProviderConfig`
|
|
167
|
-
|
|
168
|
-
:param config_keys: The configurations keys to validate
|
|
169
|
-
"""
|
|
170
|
-
if "name" not in config_keys:
|
|
171
|
-
raise ValidationError("Provider config must have name key")
|
|
172
|
-
if not any(k in config_keys for k in PLUGINS_TOPICS_KEYS):
|
|
173
|
-
raise ValidationError("A provider must implement at least one plugin")
|
|
174
|
-
non_api_keys = [k for k in PLUGINS_TOPICS_KEYS if k != "api"]
|
|
175
|
-
if "api" in config_keys and any(k in config_keys for k in non_api_keys):
|
|
176
|
-
raise ValidationError(
|
|
177
|
-
"A provider implementing an Api plugin must not implement any other "
|
|
178
|
-
"type of plugin"
|
|
179
|
-
)
|
|
180
|
-
|
|
181
|
-
def update(self, mapping: Optional[dict[str, Any]]) -> None:
|
|
182
|
-
"""Update the configuration parameters with values from `mapping`
|
|
183
|
-
|
|
184
|
-
:param mapping: The mapping from which to override configuration parameters
|
|
185
|
-
"""
|
|
186
|
-
if mapping is None:
|
|
187
|
-
mapping = {}
|
|
188
|
-
merge_mappings(
|
|
189
|
-
self.__dict__,
|
|
190
|
-
{
|
|
191
|
-
key: value
|
|
192
|
-
for key, value in mapping.items()
|
|
193
|
-
if key not in PLUGINS_TOPICS_KEYS and value is not None
|
|
194
|
-
},
|
|
195
|
-
)
|
|
196
|
-
for key in PLUGINS_TOPICS_KEYS:
|
|
197
|
-
current_value: Optional[PluginConfig] = getattr(self, key, None)
|
|
198
|
-
mapping_value = mapping.get(key, {})
|
|
199
|
-
if current_value is not None:
|
|
200
|
-
current_value.update(mapping_value)
|
|
201
|
-
elif mapping_value:
|
|
202
|
-
try:
|
|
203
|
-
setattr(self, key, PluginConfig.from_mapping(mapping_value))
|
|
204
|
-
except ValidationError as e:
|
|
205
|
-
logger.warning(
|
|
206
|
-
(
|
|
207
|
-
"Could not add %s Plugin config to %s configuration: %s. "
|
|
208
|
-
"Try updating existing %s Plugin configs instead."
|
|
209
|
-
),
|
|
210
|
-
key,
|
|
211
|
-
self.name,
|
|
212
|
-
str(e),
|
|
213
|
-
", ".join([k for k in PLUGINS_TOPICS_KEYS if hasattr(self, k)]),
|
|
214
|
-
)
|
|
215
|
-
|
|
216
|
-
|
|
217
97
|
class PluginConfig(yaml.YAMLObject):
|
|
218
98
|
"""Representation of a plugin config.
|
|
219
99
|
|
|
@@ -273,11 +153,11 @@ class PluginConfig(yaml.YAMLObject):
|
|
|
273
153
|
#: Metadata regex pattern used for discovery in search result properties
|
|
274
154
|
metadata_pattern: str
|
|
275
155
|
#: Configuration/template that will be used to query for a discovered parameter
|
|
276
|
-
search_param: str
|
|
277
|
-
#: Path to the metadata in search result
|
|
278
|
-
metadata_path: str
|
|
156
|
+
search_param: Union[str, dict[str, Any]]
|
|
279
157
|
#: list search parameters to send as is to the provider
|
|
280
158
|
search_param_unparsed: list[str]
|
|
159
|
+
#: Path to the metadata in search result
|
|
160
|
+
metadata_path: str
|
|
281
161
|
#: Use as STAC extension prefix if it does not have one already
|
|
282
162
|
metadata_prefix: str
|
|
283
163
|
#: Whether an error must be raised when using a search parameter which is not queryable or not
|
|
@@ -301,7 +181,7 @@ class PluginConfig(yaml.YAMLObject):
|
|
|
301
181
|
#: Type of the provider result
|
|
302
182
|
result_type: str
|
|
303
183
|
#: JsonPath to the list of collections
|
|
304
|
-
results_entry: Union[
|
|
184
|
+
results_entry: Union[str, JSONPath]
|
|
305
185
|
#: Mapping for the collection id
|
|
306
186
|
generic_collection_id: str
|
|
307
187
|
#: Mapping for collection metadata (e.g. ``description``, ``license``) which can be parsed from the provider
|
|
@@ -355,7 +235,6 @@ class PluginConfig(yaml.YAMLObject):
|
|
|
355
235
|
|
|
356
236
|
#: List of collection selection criterias
|
|
357
237
|
collection_selector: list[PluginConfig.CollectionSelector]
|
|
358
|
-
|
|
359
238
|
#: Configuration for queryables discovery to use
|
|
360
239
|
discover_queryables: PluginConfig.DiscoverQueryables
|
|
361
240
|
|
|
@@ -555,6 +434,8 @@ class PluginConfig(yaml.YAMLObject):
|
|
|
555
434
|
ignore_assets: bool
|
|
556
435
|
#: :class:`~eodag.plugins.download.base.Download` Collection specific configuration
|
|
557
436
|
products: dict[str, dict[str, Any]]
|
|
437
|
+
#: :class:`~eodag.plugins.download.base.Download` Number of maximum workers allowed for parallel downloads
|
|
438
|
+
max_workers: int
|
|
558
439
|
#: :class:`~eodag.plugins.download.http.HTTPDownload` Whether the product has to be ordered to download it or not
|
|
559
440
|
order_enabled: bool
|
|
560
441
|
#: :class:`~eodag.plugins.download.http.HTTPDownload` HTTP request method for the order request
|
|
@@ -575,6 +456,8 @@ class PluginConfig(yaml.YAMLObject):
|
|
|
575
456
|
#: :class:`~eodag.plugins.download.aws.AwsDownload`
|
|
576
457
|
#: At which level of the path part of the url the bucket can be found
|
|
577
458
|
bucket_path_level: int
|
|
459
|
+
#: :class:`~eodag.plugins.download.aws.AwsDownload` Whether download is done from a requester-pays bucket or not
|
|
460
|
+
requester_pays: bool
|
|
578
461
|
#: :class:`~eodag.plugins.download.aws.AwsDownload` S3 endpoint
|
|
579
462
|
s3_endpoint: str
|
|
580
463
|
|
|
@@ -601,9 +484,6 @@ class PluginConfig(yaml.YAMLObject):
|
|
|
601
484
|
#: :class:`~eodag.plugins.authentication.base.Authentication` Part of the search or download plugin configuration
|
|
602
485
|
#: that needs authentication
|
|
603
486
|
matching_conf: dict[str, Any]
|
|
604
|
-
#: :class:`~eodag.plugins.authentication.aws_auth.AwsAuth`
|
|
605
|
-
#: Whether download is done from a requester-pays bucket or not
|
|
606
|
-
requester_pays: bool
|
|
607
487
|
#: :class:`~eodag.plugins.authentication.openid_connect.OIDCRefreshTokenBase`
|
|
608
488
|
#: How the token should be used in the request
|
|
609
489
|
token_provision: str
|
|
@@ -697,39 +577,85 @@ class PluginConfig(yaml.YAMLObject):
|
|
|
697
577
|
yaml_dumper = yaml.SafeDumper
|
|
698
578
|
yaml_tag = "!plugin"
|
|
699
579
|
|
|
580
|
+
def __or__(self, other: Union[Self, dict[str, Any]]) -> Self:
|
|
581
|
+
"""Return a new PluginConfig with merged values."""
|
|
582
|
+
new_config = self.__class__.from_mapping(self.__dict__)
|
|
583
|
+
new_config.update(other)
|
|
584
|
+
return new_config
|
|
585
|
+
|
|
586
|
+
def __ior__(self, other: Union[Self, dict[str, Any]]) -> Self:
|
|
587
|
+
"""In-place update of the PluginConfig."""
|
|
588
|
+
self.update(other)
|
|
589
|
+
return self
|
|
590
|
+
|
|
591
|
+
def __contains__(self, item: str) -> bool:
|
|
592
|
+
"""Check if a key is in the PluginConfig."""
|
|
593
|
+
return item in self.__dict__
|
|
594
|
+
|
|
700
595
|
@classmethod
|
|
701
|
-
def from_yaml(cls, loader: yaml.Loader, node: Any) ->
|
|
596
|
+
def from_yaml(cls, loader: yaml.Loader, node: Any) -> Self:
|
|
702
597
|
"""Build a :class:`~eodag.config.PluginConfig` from Yaml"""
|
|
703
598
|
cls.validate(tuple(node_key.value for node_key, _ in node.value))
|
|
704
599
|
return loader.construct_yaml_object(node, cls)
|
|
705
600
|
|
|
706
601
|
@classmethod
|
|
707
|
-
def from_mapping(cls, mapping: dict[str, Any]) ->
|
|
602
|
+
def from_mapping(cls, mapping: dict[str, Any]) -> Self:
|
|
708
603
|
"""Build a :class:`~eodag.config.PluginConfig` from a mapping"""
|
|
709
604
|
cls.validate(tuple(mapping.keys()))
|
|
710
605
|
c = cls()
|
|
711
|
-
c.__dict__.update(mapping)
|
|
606
|
+
c.__dict__.update(deepcopy(mapping))
|
|
712
607
|
return c
|
|
713
608
|
|
|
714
609
|
@staticmethod
|
|
715
610
|
def validate(config_keys: tuple[Any, ...]) -> None:
|
|
716
611
|
"""Validate a :class:`~eodag.config.PluginConfig`"""
|
|
717
|
-
|
|
612
|
+
# credentials may be set without type when the provider uses search_auth plugin.
|
|
613
|
+
if "type" not in config_keys and "credentials" not in config_keys:
|
|
718
614
|
raise ValidationError(
|
|
719
615
|
"A Plugin config must specify the type of Plugin it configures"
|
|
720
616
|
)
|
|
721
617
|
|
|
722
|
-
def update(self,
|
|
618
|
+
def update(self, config: Optional[Union[Self, dict[Any, Any]]]) -> None:
|
|
723
619
|
"""Update the configuration parameters with values from `mapping`
|
|
724
620
|
|
|
725
621
|
:param mapping: The mapping from which to override configuration parameters
|
|
726
622
|
"""
|
|
727
|
-
if
|
|
728
|
-
|
|
623
|
+
if config is None:
|
|
624
|
+
return
|
|
625
|
+
source = config if isinstance(config, dict) else config.__dict__
|
|
729
626
|
merge_mappings(
|
|
730
|
-
self.__dict__, {k: v for k, v in
|
|
627
|
+
self.__dict__, {k: v for k, v in source.items() if v is not None}
|
|
731
628
|
)
|
|
732
629
|
|
|
630
|
+
def matches_target_auth(self, target_config: Self):
|
|
631
|
+
"""Check if the target auth configuration matches this one"""
|
|
632
|
+
target_matching_conf = getattr(target_config, "matching_conf", {})
|
|
633
|
+
target_matching_url = getattr(target_config, "matching_url", None)
|
|
634
|
+
|
|
635
|
+
matching_conf = getattr(self, "matching_conf", {})
|
|
636
|
+
matching_url = getattr(self, "matching_url", None)
|
|
637
|
+
|
|
638
|
+
if target_matching_conf and sort_dict(target_matching_conf) == sort_dict(
|
|
639
|
+
matching_conf
|
|
640
|
+
):
|
|
641
|
+
return True
|
|
642
|
+
|
|
643
|
+
if target_matching_url and target_matching_url == matching_url:
|
|
644
|
+
return True
|
|
645
|
+
|
|
646
|
+
return False
|
|
647
|
+
|
|
648
|
+
|
|
649
|
+
def credentials_in_auth(auth_conf: PluginConfig) -> bool:
|
|
650
|
+
"""Checks if credentials are set for this Authentication plugin configuration
|
|
651
|
+
|
|
652
|
+
:param auth_conf: Authentication plugin configuration
|
|
653
|
+
:returns: True if credentials are set, else False
|
|
654
|
+
"""
|
|
655
|
+
return any(
|
|
656
|
+
c is not None for c in (getattr(auth_conf, "credentials", {}) or {}).values()
|
|
657
|
+
)
|
|
658
|
+
|
|
733
659
|
|
|
734
660
|
def load_default_config() -> dict[str, ProviderConfig]:
|
|
735
661
|
"""Load the providers configuration into a dictionary.
|
|
@@ -742,6 +668,7 @@ def load_default_config() -> dict[str, ProviderConfig]:
|
|
|
742
668
|
eodag_providers_cfg_file = os.getenv("EODAG_PROVIDERS_CFG_FILE") or str(
|
|
743
669
|
res_files("eodag") / "resources" / "providers.yml"
|
|
744
670
|
)
|
|
671
|
+
|
|
745
672
|
return load_config(eodag_providers_cfg_file)
|
|
746
673
|
|
|
747
674
|
|
|
@@ -754,7 +681,7 @@ def load_config(config_path: str) -> dict[str, ProviderConfig]:
|
|
|
754
681
|
:returns: The default provider's configuration
|
|
755
682
|
"""
|
|
756
683
|
logger.debug("Loading configuration from %s", config_path)
|
|
757
|
-
|
|
684
|
+
|
|
758
685
|
try:
|
|
759
686
|
# Providers configs are stored in this file as separated yaml documents
|
|
760
687
|
# Load all of it
|
|
@@ -763,305 +690,7 @@ def load_config(config_path: str) -> dict[str, ProviderConfig]:
|
|
|
763
690
|
logger.error("Unable to load configuration")
|
|
764
691
|
raise e
|
|
765
692
|
|
|
766
|
-
|
|
767
|
-
|
|
768
|
-
whitelist_env = os.getenv("EODAG_PROVIDERS_WHITELIST")
|
|
769
|
-
whitelist = None
|
|
770
|
-
if whitelist_env:
|
|
771
|
-
whitelist = {provider for provider in whitelist_env.split(",")}
|
|
772
|
-
logger.info("Using providers whitelist: %s", ", ".join(whitelist))
|
|
773
|
-
|
|
774
|
-
for provider_config in providers_configs:
|
|
775
|
-
if provider_config is None or (
|
|
776
|
-
whitelist and provider_config.name not in whitelist
|
|
777
|
-
):
|
|
778
|
-
continue
|
|
779
|
-
provider_config_init(provider_config, stac_provider_config)
|
|
780
|
-
config[provider_config.name] = provider_config
|
|
781
|
-
|
|
782
|
-
return config
|
|
783
|
-
|
|
784
|
-
|
|
785
|
-
def credentials_in_auth(auth_conf: PluginConfig) -> bool:
|
|
786
|
-
"""Checks if credentials are set for this Authentication plugin configuration
|
|
787
|
-
|
|
788
|
-
:param auth_conf: Authentication plugin configuration
|
|
789
|
-
:returns: True if credentials are set, else False
|
|
790
|
-
"""
|
|
791
|
-
return any(
|
|
792
|
-
c is not None for c in (getattr(auth_conf, "credentials", {}) or {}).values()
|
|
793
|
-
)
|
|
794
|
-
|
|
795
|
-
|
|
796
|
-
def share_credentials(
|
|
797
|
-
providers_config: dict[str, ProviderConfig],
|
|
798
|
-
) -> None:
|
|
799
|
-
"""Share credentials between plugins having the same matching criteria
|
|
800
|
-
|
|
801
|
-
:param providers_configs: eodag providers configurations
|
|
802
|
-
"""
|
|
803
|
-
auth_confs_with_creds = [
|
|
804
|
-
getattr(p, k)
|
|
805
|
-
for p in providers_config.values()
|
|
806
|
-
for k in AUTH_TOPIC_KEYS
|
|
807
|
-
if hasattr(p, k) and credentials_in_auth(getattr(p, k))
|
|
808
|
-
]
|
|
809
|
-
for provider, provider_config in providers_config.items():
|
|
810
|
-
if auth_confs_with_creds:
|
|
811
|
-
for auth_topic_key in AUTH_TOPIC_KEYS:
|
|
812
|
-
provider_config_auth = getattr(provider_config, auth_topic_key, None)
|
|
813
|
-
if provider_config_auth and not credentials_in_auth(
|
|
814
|
-
provider_config_auth
|
|
815
|
-
):
|
|
816
|
-
# no credentials set for this provider
|
|
817
|
-
provider_matching_conf = getattr(
|
|
818
|
-
provider_config_auth, "matching_conf", {}
|
|
819
|
-
)
|
|
820
|
-
provider_matching_url = getattr(
|
|
821
|
-
provider_config_auth, "matching_url", None
|
|
822
|
-
)
|
|
823
|
-
for conf_with_creds in auth_confs_with_creds:
|
|
824
|
-
# copy credentials between plugins if `matching_conf` or `matching_url` are matching
|
|
825
|
-
if (
|
|
826
|
-
provider_matching_conf
|
|
827
|
-
and sort_dict(provider_matching_conf)
|
|
828
|
-
== sort_dict(getattr(conf_with_creds, "matching_conf", {}))
|
|
829
|
-
) or (
|
|
830
|
-
provider_matching_url
|
|
831
|
-
and provider_matching_url
|
|
832
|
-
== getattr(conf_with_creds, "matching_url", None)
|
|
833
|
-
):
|
|
834
|
-
getattr(
|
|
835
|
-
providers_config[provider], auth_topic_key
|
|
836
|
-
).credentials = conf_with_creds.credentials
|
|
837
|
-
|
|
838
|
-
|
|
839
|
-
def provider_config_init(
|
|
840
|
-
provider_config: ProviderConfig,
|
|
841
|
-
stac_search_default_conf: Optional[dict[str, Any]] = None,
|
|
842
|
-
) -> None:
|
|
843
|
-
"""Applies some default values to provider config
|
|
844
|
-
|
|
845
|
-
:param provider_config: An eodag provider configuration
|
|
846
|
-
:param stac_search_default_conf: default conf to overwrite with provider_config if STAC
|
|
847
|
-
"""
|
|
848
|
-
# For the provider, set the default output_dir of its download plugin
|
|
849
|
-
# as tempdir in a portable way
|
|
850
|
-
for download_topic_key in ("download", "api"):
|
|
851
|
-
if download_topic_key in vars(provider_config):
|
|
852
|
-
download_conf = getattr(provider_config, download_topic_key)
|
|
853
|
-
if not getattr(download_conf, "output_dir", None):
|
|
854
|
-
download_conf.output_dir = tempfile.gettempdir()
|
|
855
|
-
if not getattr(download_conf, "delete_archive", None):
|
|
856
|
-
download_conf.delete_archive = True
|
|
857
|
-
|
|
858
|
-
try:
|
|
859
|
-
if (
|
|
860
|
-
stac_search_default_conf is not None
|
|
861
|
-
and provider_config.search
|
|
862
|
-
and provider_config.search.type in STAC_SEARCH_PLUGINS
|
|
863
|
-
):
|
|
864
|
-
# search config set to stac defaults overriden with provider config
|
|
865
|
-
per_provider_stac_provider_config = deepcopy(stac_search_default_conf)
|
|
866
|
-
provider_config.search.__dict__ = update_nested_dict(
|
|
867
|
-
per_provider_stac_provider_config["search"],
|
|
868
|
-
provider_config.search.__dict__,
|
|
869
|
-
allow_empty_values=True,
|
|
870
|
-
)
|
|
871
|
-
except AttributeError:
|
|
872
|
-
pass
|
|
873
|
-
|
|
874
|
-
|
|
875
|
-
def override_config_from_file(
|
|
876
|
-
config: dict[str, ProviderConfig], file_path: str
|
|
877
|
-
) -> None:
|
|
878
|
-
"""Override a configuration with the values in a file
|
|
879
|
-
|
|
880
|
-
:param config: An eodag providers configuration dictionary
|
|
881
|
-
:param file_path: The path to the file from where the new values will be read
|
|
882
|
-
"""
|
|
883
|
-
logger.info("Loading user configuration from: %s", os.path.abspath(file_path))
|
|
884
|
-
with open(os.path.abspath(os.path.realpath(file_path)), "r") as fh:
|
|
885
|
-
try:
|
|
886
|
-
config_in_file = yaml.safe_load(fh)
|
|
887
|
-
if config_in_file is None:
|
|
888
|
-
return
|
|
889
|
-
except yaml.parser.ParserError as e:
|
|
890
|
-
logger.error("Unable to load user configuration file")
|
|
891
|
-
raise e
|
|
892
|
-
|
|
893
|
-
override_config_from_mapping(config, config_in_file)
|
|
894
|
-
|
|
895
|
-
|
|
896
|
-
def override_config_from_env(config: dict[str, ProviderConfig]) -> None:
|
|
897
|
-
"""Override a configuration with environment variables values
|
|
898
|
-
|
|
899
|
-
:param config: An eodag providers configuration dictionary
|
|
900
|
-
"""
|
|
901
|
-
|
|
902
|
-
def build_mapping_from_env(
|
|
903
|
-
env_var: str, env_value: str, mapping: dict[str, Any]
|
|
904
|
-
) -> None:
|
|
905
|
-
"""Recursively build a dictionary from an environment variable.
|
|
906
|
-
|
|
907
|
-
The environment variable must respect the pattern: KEY1__KEY2__[...]__KEYN.
|
|
908
|
-
It will be transformed to::
|
|
909
|
-
|
|
910
|
-
{
|
|
911
|
-
"key1": {
|
|
912
|
-
"key2": {
|
|
913
|
-
{...}
|
|
914
|
-
}
|
|
915
|
-
}
|
|
916
|
-
}
|
|
917
|
-
|
|
918
|
-
:param env_var: The environment variable to be transformed into a dictionary
|
|
919
|
-
:param env_value: The value from environment variable
|
|
920
|
-
:param mapping: The mapping in which the value will be created
|
|
921
|
-
"""
|
|
922
|
-
parts = env_var.split("__")
|
|
923
|
-
iter_parts = iter(parts)
|
|
924
|
-
env_type = get_type_hints(PluginConfig).get(next(iter_parts, ""), str)
|
|
925
|
-
child_env_type = (
|
|
926
|
-
get_type_hints(env_type).get(next(iter_parts, ""))
|
|
927
|
-
if isclass(env_type)
|
|
928
|
-
else None
|
|
929
|
-
)
|
|
930
|
-
if len(parts) == 2 and child_env_type:
|
|
931
|
-
# for nested config (pagination, ...)
|
|
932
|
-
# try converting env_value type from type hints
|
|
933
|
-
try:
|
|
934
|
-
env_value = cast_scalar_value(env_value, child_env_type)
|
|
935
|
-
except TypeError:
|
|
936
|
-
logger.warning(
|
|
937
|
-
f"Could not convert {parts} value {env_value} to {child_env_type}"
|
|
938
|
-
)
|
|
939
|
-
mapping.setdefault(parts[0], {})
|
|
940
|
-
mapping[parts[0]][parts[1]] = env_value
|
|
941
|
-
elif len(parts) == 1:
|
|
942
|
-
# try converting env_value type from type hints
|
|
943
|
-
try:
|
|
944
|
-
env_value = cast_scalar_value(env_value, env_type)
|
|
945
|
-
except TypeError:
|
|
946
|
-
logger.warning(
|
|
947
|
-
f"Could not convert {parts[0]} value {env_value} to {env_type}"
|
|
948
|
-
)
|
|
949
|
-
mapping[parts[0]] = env_value
|
|
950
|
-
else:
|
|
951
|
-
new_map = mapping.setdefault(parts[0], {})
|
|
952
|
-
build_mapping_from_env("__".join(parts[1:]), env_value, new_map)
|
|
953
|
-
|
|
954
|
-
mapping_from_env: dict[str, Any] = {}
|
|
955
|
-
for env_var in os.environ:
|
|
956
|
-
if env_var.startswith("EODAG__"):
|
|
957
|
-
build_mapping_from_env(
|
|
958
|
-
env_var[len("EODAG__") :].lower(), # noqa
|
|
959
|
-
os.environ[env_var],
|
|
960
|
-
mapping_from_env,
|
|
961
|
-
)
|
|
962
|
-
|
|
963
|
-
override_config_from_mapping(config, mapping_from_env)
|
|
964
|
-
|
|
965
|
-
|
|
966
|
-
def override_config_from_mapping(
|
|
967
|
-
config: dict[str, ProviderConfig], mapping: dict[str, Any]
|
|
968
|
-
) -> None:
|
|
969
|
-
"""Override a configuration with the values in a mapping.
|
|
970
|
-
|
|
971
|
-
If the environment variable ``EODAG_PROVIDERS_WHITELIST`` is set (as a comma-separated list of provider names),
|
|
972
|
-
only the listed providers will be used from the mapping. All other providers in the mapping will be ignored.
|
|
973
|
-
|
|
974
|
-
:param config: An eodag providers configuration dictionary
|
|
975
|
-
:param mapping: The mapping containing the values to be overriden
|
|
976
|
-
"""
|
|
977
|
-
whitelist_env = os.getenv("EODAG_PROVIDERS_WHITELIST")
|
|
978
|
-
whitelist = None
|
|
979
|
-
if whitelist_env:
|
|
980
|
-
whitelist = {provider for provider in whitelist_env.split(",")}
|
|
981
|
-
|
|
982
|
-
for provider, new_conf in mapping.items():
|
|
983
|
-
# check if metada-mapping as already been built as jsonpath in providers_config
|
|
984
|
-
# or provider not in whitelist
|
|
985
|
-
if not isinstance(new_conf, dict) or (whitelist and provider not in whitelist):
|
|
986
|
-
continue
|
|
987
|
-
new_conf_search = new_conf.get("search", {}) or {}
|
|
988
|
-
new_conf_api = new_conf.get("api", {}) or {}
|
|
989
|
-
if provider in config and "metadata_mapping" in {
|
|
990
|
-
**new_conf_search,
|
|
991
|
-
**new_conf_api,
|
|
992
|
-
}:
|
|
993
|
-
search_plugin_key = (
|
|
994
|
-
"search" if "metadata_mapping" in new_conf_search else "api"
|
|
995
|
-
)
|
|
996
|
-
# get some already configured value
|
|
997
|
-
configured_metadata_mapping = getattr(
|
|
998
|
-
config[provider], search_plugin_key
|
|
999
|
-
).metadata_mapping
|
|
1000
|
-
some_configured_value = next(iter(configured_metadata_mapping.values()))
|
|
1001
|
-
# check if the configured value has already been built as jsonpath
|
|
1002
|
-
if (
|
|
1003
|
-
isinstance(some_configured_value, list)
|
|
1004
|
-
and isinstance(some_configured_value[1], tuple)
|
|
1005
|
-
or isinstance(some_configured_value, tuple)
|
|
1006
|
-
):
|
|
1007
|
-
# also build as jsonpath the incoming conf
|
|
1008
|
-
mtd_cfg_as_conversion_and_querypath(
|
|
1009
|
-
deepcopy(mapping[provider][search_plugin_key]["metadata_mapping"]),
|
|
1010
|
-
mapping[provider][search_plugin_key]["metadata_mapping"],
|
|
1011
|
-
)
|
|
1012
|
-
|
|
1013
|
-
# try overriding conf
|
|
1014
|
-
old_conf: Optional[ProviderConfig] = config.get(provider)
|
|
1015
|
-
if old_conf is not None:
|
|
1016
|
-
old_conf.update(new_conf)
|
|
1017
|
-
else:
|
|
1018
|
-
logger.info(
|
|
1019
|
-
"%s: unknown provider found in user conf, trying to use provided configuration",
|
|
1020
|
-
provider,
|
|
1021
|
-
)
|
|
1022
|
-
try:
|
|
1023
|
-
new_conf["name"] = new_conf.get("name", provider)
|
|
1024
|
-
config[provider] = ProviderConfig.from_mapping(new_conf)
|
|
1025
|
-
except Exception:
|
|
1026
|
-
logger.warning(
|
|
1027
|
-
"%s skipped: could not be loaded from user configuration", provider
|
|
1028
|
-
)
|
|
1029
|
-
|
|
1030
|
-
import traceback as tb
|
|
1031
|
-
|
|
1032
|
-
logger.debug(tb.format_exc())
|
|
1033
|
-
|
|
1034
|
-
|
|
1035
|
-
def merge_configs(config: dict[str, Any], other_config: dict[str, Any]) -> None:
|
|
1036
|
-
"""Override a configuration with the values of another configuration
|
|
1037
|
-
|
|
1038
|
-
:param config: An eodag providers configuration dictionary
|
|
1039
|
-
:param other_config: An eodag providers configuration dictionary
|
|
1040
|
-
"""
|
|
1041
|
-
# configs union with other_config values as default
|
|
1042
|
-
other_config = dict(config, **other_config)
|
|
1043
|
-
|
|
1044
|
-
for provider, new_conf in other_config.items():
|
|
1045
|
-
old_conf = config.get(provider)
|
|
1046
|
-
|
|
1047
|
-
if old_conf:
|
|
1048
|
-
# update non-objects values
|
|
1049
|
-
new_conf = dict(old_conf.__dict__, **new_conf.__dict__)
|
|
1050
|
-
|
|
1051
|
-
for conf_k, conf_v in new_conf.items():
|
|
1052
|
-
old_conf_v = getattr(old_conf, conf_k, None)
|
|
1053
|
-
|
|
1054
|
-
if isinstance(conf_v, PluginConfig) and isinstance(
|
|
1055
|
-
old_conf_v, PluginConfig
|
|
1056
|
-
):
|
|
1057
|
-
old_conf_v.update(conf_v.__dict__)
|
|
1058
|
-
new_conf[conf_k] = old_conf_v
|
|
1059
|
-
elif isinstance(old_conf_v, PluginConfig):
|
|
1060
|
-
new_conf[conf_k] = old_conf_v
|
|
1061
|
-
|
|
1062
|
-
setattr(config[provider], conf_k, new_conf[conf_k])
|
|
1063
|
-
else:
|
|
1064
|
-
config[provider] = new_conf
|
|
693
|
+
return {p.name: p for p in providers_configs if p is not None}
|
|
1065
694
|
|
|
1066
695
|
|
|
1067
696
|
def load_yml_config(yml_path: str) -> dict[Any, Any]:
|
|
@@ -226,7 +226,6 @@ class TokenAuth(Authentication):
|
|
|
226
226
|
self,
|
|
227
227
|
session: requests.Session,
|
|
228
228
|
) -> requests.Response:
|
|
229
|
-
|
|
230
229
|
retry_total = getattr(self.config, "retry_total", REQ_RETRY_TOTAL)
|
|
231
230
|
retry_backoff_factor = getattr(
|
|
232
231
|
self.config, "retry_backoff_factor", REQ_RETRY_BACKOFF_FACTOR
|
eodag/plugins/download/http.py
CHANGED
|
@@ -155,7 +155,6 @@ class HTTPDownload(Download):
|
|
|
155
155
|
auth: Optional[AuthBase] = None,
|
|
156
156
|
**kwargs: Unpack[DownloadConf],
|
|
157
157
|
) -> Optional[dict[str, Any]]:
|
|
158
|
-
|
|
159
158
|
"""Send product order request.
|
|
160
159
|
|
|
161
160
|
It will be executed once before the download retry loop, if the product is orderable
|