eodag 3.4.3__py3-none-any.whl → 3.5.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- eodag/api/core.py +75 -1
- eodag/api/product/metadata_mapping.py +31 -2
- eodag/config.py +56 -10
- eodag/plugins/authentication/keycloak.py +3 -1
- eodag/plugins/authentication/openid_connect.py +21 -3
- eodag/plugins/authentication/token.py +14 -2
- eodag/plugins/authentication/token_exchange.py +1 -0
- eodag/plugins/download/aws.py +1 -1
- eodag/plugins/manager.py +1 -1
- eodag/plugins/search/base.py +3 -2
- eodag/plugins/search/build_search_result.py +10 -4
- eodag/plugins/search/data_request_search.py +14 -1
- eodag/plugins/search/qssearch.py +8 -2
- eodag/resources/ext_product_types.json +1 -1
- eodag/resources/providers.yml +4 -0
- eodag/rest/core.py +3 -1
- eodag/rest/errors.py +9 -4
- eodag/rest/server.py +3 -1
- eodag/rest/stac.py +22 -1
- eodag/utils/__init__.py +3 -0
- eodag/utils/env.py +29 -0
- {eodag-3.4.3.dist-info → eodag-3.5.0.dist-info}/METADATA +3 -3
- {eodag-3.4.3.dist-info → eodag-3.5.0.dist-info}/RECORD +27 -26
- {eodag-3.4.3.dist-info → eodag-3.5.0.dist-info}/WHEEL +0 -0
- {eodag-3.4.3.dist-info → eodag-3.5.0.dist-info}/entry_points.txt +0 -0
- {eodag-3.4.3.dist-info → eodag-3.5.0.dist-info}/licenses/LICENSE +0 -0
- {eodag-3.4.3.dist-info → eodag-3.5.0.dist-info}/top_level.txt +0 -0
eodag/api/core.py
CHANGED
|
@@ -36,6 +36,7 @@ from whoosh.index import exists_in, open_dir
|
|
|
36
36
|
from whoosh.qparser import QueryParser
|
|
37
37
|
|
|
38
38
|
from eodag.api.product.metadata_mapping import (
|
|
39
|
+
NOT_AVAILABLE,
|
|
39
40
|
ONLINE_STATUS,
|
|
40
41
|
mtd_cfg_as_conversion_and_querypath,
|
|
41
42
|
)
|
|
@@ -79,6 +80,7 @@ from eodag.utils import (
|
|
|
79
80
|
string_to_jsonpath,
|
|
80
81
|
uri_to_path,
|
|
81
82
|
)
|
|
83
|
+
from eodag.utils.env import is_env_var_true
|
|
82
84
|
from eodag.utils.exceptions import (
|
|
83
85
|
AuthenticationError,
|
|
84
86
|
EodagError,
|
|
@@ -175,12 +177,19 @@ class EODataAccessGateway:
|
|
|
175
177
|
share_credentials(self.providers_config)
|
|
176
178
|
|
|
177
179
|
# init updated providers conf
|
|
180
|
+
strict_mode = is_env_var_true("EODAG_STRICT_PRODUCT_TYPES")
|
|
181
|
+
available_product_types = set(self.product_types_config.source.keys())
|
|
182
|
+
|
|
178
183
|
for provider in self.providers_config.keys():
|
|
179
184
|
provider_config_init(
|
|
180
185
|
self.providers_config[provider],
|
|
181
186
|
load_stac_provider_config(),
|
|
182
187
|
)
|
|
183
188
|
|
|
189
|
+
self._sync_provider_product_types(
|
|
190
|
+
provider, available_product_types, strict_mode
|
|
191
|
+
)
|
|
192
|
+
|
|
184
193
|
# re-build _plugins_manager using up-to-date providers_config
|
|
185
194
|
self._plugins_manager.rebuild(self.providers_config)
|
|
186
195
|
|
|
@@ -226,6 +235,61 @@ class EODataAccessGateway:
|
|
|
226
235
|
)
|
|
227
236
|
self.set_locations_conf(locations_conf_path)
|
|
228
237
|
|
|
238
|
+
def _sync_provider_product_types(
|
|
239
|
+
self,
|
|
240
|
+
provider: str,
|
|
241
|
+
available_product_types: set[str],
|
|
242
|
+
strict_mode: bool,
|
|
243
|
+
) -> None:
|
|
244
|
+
"""
|
|
245
|
+
Synchronize product types for a provider based on strict or permissive mode.
|
|
246
|
+
|
|
247
|
+
In strict mode, removes product types not in available_product_types.
|
|
248
|
+
In permissive mode, adds empty product type configs for missing types.
|
|
249
|
+
|
|
250
|
+
:param provider: The provider name whose product types should be synchronized.
|
|
251
|
+
:param available_product_types: The set of available product type IDs.
|
|
252
|
+
:param strict_mode: If True, remove unknown product types; if False, add empty configs for them.
|
|
253
|
+
:returns: None
|
|
254
|
+
"""
|
|
255
|
+
provider_products = self.providers_config[provider].products
|
|
256
|
+
products_to_remove: list[str] = []
|
|
257
|
+
products_to_add: list[str] = []
|
|
258
|
+
|
|
259
|
+
for product_id in provider_products:
|
|
260
|
+
if product_id == GENERIC_PRODUCT_TYPE:
|
|
261
|
+
continue
|
|
262
|
+
|
|
263
|
+
if product_id not in available_product_types:
|
|
264
|
+
if strict_mode:
|
|
265
|
+
products_to_remove.append(product_id)
|
|
266
|
+
continue
|
|
267
|
+
|
|
268
|
+
empty_product = {
|
|
269
|
+
"title": product_id,
|
|
270
|
+
"abstract": NOT_AVAILABLE,
|
|
271
|
+
}
|
|
272
|
+
self.product_types_config.source[
|
|
273
|
+
product_id
|
|
274
|
+
] = empty_product # will update available_product_types
|
|
275
|
+
products_to_add.append(product_id)
|
|
276
|
+
|
|
277
|
+
if products_to_add:
|
|
278
|
+
logger.debug(
|
|
279
|
+
"Product types permissive mode, %s added (provider %s)",
|
|
280
|
+
", ".join(products_to_add),
|
|
281
|
+
provider,
|
|
282
|
+
)
|
|
283
|
+
|
|
284
|
+
if products_to_remove:
|
|
285
|
+
logger.debug(
|
|
286
|
+
"Product types strict mode, ignoring %s (provider %s)",
|
|
287
|
+
", ".join(products_to_remove),
|
|
288
|
+
provider,
|
|
289
|
+
)
|
|
290
|
+
for id in products_to_remove:
|
|
291
|
+
del self.providers_config[provider].products[id]
|
|
292
|
+
|
|
229
293
|
def get_version(self) -> str:
|
|
230
294
|
"""Get eodag package version"""
|
|
231
295
|
return version("eodag")
|
|
@@ -608,11 +672,14 @@ class EODataAccessGateway:
|
|
|
608
672
|
for product_type_id in p.products: # type: ignore
|
|
609
673
|
if product_type_id == GENERIC_PRODUCT_TYPE:
|
|
610
674
|
continue
|
|
675
|
+
|
|
611
676
|
config = self.product_types_config[product_type_id]
|
|
612
677
|
config["_id"] = product_type_id
|
|
678
|
+
|
|
613
679
|
if "alias" in config:
|
|
614
680
|
product_type_id = config["alias"]
|
|
615
681
|
product_type = {"ID": product_type_id, **config}
|
|
682
|
+
|
|
616
683
|
if product_type not in product_types:
|
|
617
684
|
product_types.append(product_type)
|
|
618
685
|
|
|
@@ -620,11 +687,18 @@ class EODataAccessGateway:
|
|
|
620
687
|
return sorted(product_types, key=itemgetter("ID"))
|
|
621
688
|
|
|
622
689
|
def fetch_product_types_list(self, provider: Optional[str] = None) -> None:
|
|
623
|
-
"""Fetch product types list and update if needed
|
|
690
|
+
"""Fetch product types list and update if needed.
|
|
691
|
+
|
|
692
|
+
If strict mode is enabled (by setting the ``EODAG_STRICT_PRODUCT_TYPES`` environment variable
|
|
693
|
+
to a truthy value), this method will not fetch or update product types and will return immediately.
|
|
624
694
|
|
|
625
695
|
:param provider: The name of a provider or provider-group for which product types
|
|
626
696
|
list should be updated. Defaults to all providers (None value).
|
|
627
697
|
"""
|
|
698
|
+
strict_mode = is_env_var_true("EODAG_STRICT_PRODUCT_TYPES")
|
|
699
|
+
if strict_mode:
|
|
700
|
+
return
|
|
701
|
+
|
|
628
702
|
providers_to_fetch = list(self.providers_config.keys())
|
|
629
703
|
# check if some providers are grouped under a group name which is not a provider name
|
|
630
704
|
if provider is not None and provider not in self.providers_config:
|
|
@@ -54,6 +54,7 @@ from eodag.utils import (
|
|
|
54
54
|
string_to_jsonpath,
|
|
55
55
|
update_nested_dict,
|
|
56
56
|
)
|
|
57
|
+
from eodag.utils.exceptions import ValidationError
|
|
57
58
|
|
|
58
59
|
if TYPE_CHECKING:
|
|
59
60
|
from shapely.geometry.base import BaseGeometry
|
|
@@ -1286,7 +1287,10 @@ def mtd_cfg_as_conversion_and_querypath(
|
|
|
1286
1287
|
|
|
1287
1288
|
|
|
1288
1289
|
def format_query_params(
|
|
1289
|
-
product_type: str,
|
|
1290
|
+
product_type: str,
|
|
1291
|
+
config: PluginConfig,
|
|
1292
|
+
query_dict: dict[str, Any],
|
|
1293
|
+
error_context: str = "",
|
|
1290
1294
|
) -> dict[str, Any]:
|
|
1291
1295
|
"""format the search parameters to query parameters"""
|
|
1292
1296
|
if "raise_errors" in query_dict.keys():
|
|
@@ -1299,10 +1303,26 @@ def format_query_params(
|
|
|
1299
1303
|
**config.products.get(product_type, {}).get("metadata_mapping", {}),
|
|
1300
1304
|
)
|
|
1301
1305
|
|
|
1306
|
+
# Raise error if non-queryables parameters are used and raise_mtd_discovery_error configured
|
|
1307
|
+
if (
|
|
1308
|
+
raise_mtd_discovery_error := config.products.get(product_type, {})
|
|
1309
|
+
.get("discover_metadata", {})
|
|
1310
|
+
.get("raise_mtd_discovery_error")
|
|
1311
|
+
) is None:
|
|
1312
|
+
raise_mtd_discovery_error = getattr(config, "discover_metadata", {}).get(
|
|
1313
|
+
"raise_mtd_discovery_error", False
|
|
1314
|
+
)
|
|
1315
|
+
|
|
1302
1316
|
query_params: dict[str, Any] = {}
|
|
1303
1317
|
# Get all the search parameters that are recognised as queryables by the
|
|
1304
1318
|
# provider (they appear in the queryables dictionary)
|
|
1305
|
-
queryables = _get_queryables(
|
|
1319
|
+
queryables = _get_queryables(
|
|
1320
|
+
query_dict,
|
|
1321
|
+
config,
|
|
1322
|
+
product_type_metadata_mapping,
|
|
1323
|
+
raise_mtd_discovery_error,
|
|
1324
|
+
error_context,
|
|
1325
|
+
)
|
|
1306
1326
|
|
|
1307
1327
|
for eodag_search_key, provider_search_key in queryables.items():
|
|
1308
1328
|
user_input = query_dict[eodag_search_key]
|
|
@@ -1438,6 +1458,8 @@ def _get_queryables(
|
|
|
1438
1458
|
search_params: dict[str, Any],
|
|
1439
1459
|
config: PluginConfig,
|
|
1440
1460
|
metadata_mapping: dict[str, Any],
|
|
1461
|
+
raise_mtd_discovery_error: bool,
|
|
1462
|
+
error_context: str,
|
|
1441
1463
|
) -> dict[str, Any]:
|
|
1442
1464
|
"""Retrieve the metadata mappings that are query-able"""
|
|
1443
1465
|
logger.debug("Retrieving queryable metadata from metadata_mapping")
|
|
@@ -1445,6 +1467,13 @@ def _get_queryables(
|
|
|
1445
1467
|
for eodag_search_key, user_input in search_params.items():
|
|
1446
1468
|
if user_input is not None:
|
|
1447
1469
|
md_mapping = metadata_mapping.get(eodag_search_key, (None, NOT_MAPPED))
|
|
1470
|
+
# raise an error when a query param not allowed by the provider is found
|
|
1471
|
+
if not isinstance(md_mapping, list) and raise_mtd_discovery_error:
|
|
1472
|
+
raise ValidationError(
|
|
1473
|
+
"Search parameters which are not queryable are disallowed for this product type on this provider: "
|
|
1474
|
+
f"please remove '{eodag_search_key}' from your search parameters. {error_context}",
|
|
1475
|
+
{eodag_search_key},
|
|
1476
|
+
)
|
|
1448
1477
|
_, md_value = md_mapping
|
|
1449
1478
|
# query param from defined metadata_mapping
|
|
1450
1479
|
if md_mapping is not None and isinstance(md_mapping, list):
|
eodag/config.py
CHANGED
|
@@ -194,12 +194,24 @@ class ProviderConfig(yaml.YAMLObject):
|
|
|
194
194
|
},
|
|
195
195
|
)
|
|
196
196
|
for key in PLUGINS_TOPICS_KEYS:
|
|
197
|
-
current_value: Optional[
|
|
197
|
+
current_value: Optional[PluginConfig] = getattr(self, key, None)
|
|
198
198
|
mapping_value = mapping.get(key, {})
|
|
199
199
|
if current_value is not None:
|
|
200
200
|
current_value.update(mapping_value)
|
|
201
201
|
elif mapping_value:
|
|
202
|
-
|
|
202
|
+
try:
|
|
203
|
+
setattr(self, key, PluginConfig.from_mapping(mapping_value))
|
|
204
|
+
except ValidationError as e:
|
|
205
|
+
logger.warning(
|
|
206
|
+
(
|
|
207
|
+
"Could not add %s Plugin config to %s configuration: %s. "
|
|
208
|
+
"Try updating existing %s Plugin configs instead."
|
|
209
|
+
),
|
|
210
|
+
key,
|
|
211
|
+
self.name,
|
|
212
|
+
str(e),
|
|
213
|
+
", ".join([k for k in PLUGINS_TOPICS_KEYS if hasattr(self, k)]),
|
|
214
|
+
)
|
|
203
215
|
|
|
204
216
|
|
|
205
217
|
class PluginConfig(yaml.YAMLObject):
|
|
@@ -256,9 +268,11 @@ class PluginConfig(yaml.YAMLObject):
|
|
|
256
268
|
#: Metadata regex pattern used for discovery in search result properties
|
|
257
269
|
metadata_pattern: str
|
|
258
270
|
#: Configuration/template that will be used to query for a discovered parameter
|
|
259
|
-
search_param: str
|
|
271
|
+
search_param: str | dict[str, Any]
|
|
260
272
|
#: Path to the metadata in search result
|
|
261
273
|
metadata_path: str
|
|
274
|
+
#: Whether an error must be raised when using a search parameter which is not queryable or not
|
|
275
|
+
raise_mtd_discovery_error: bool
|
|
262
276
|
|
|
263
277
|
class DiscoverProductTypes(TypedDict, total=False):
|
|
264
278
|
"""Configuration for product types discovery"""
|
|
@@ -628,6 +642,10 @@ class PluginConfig(yaml.YAMLObject):
|
|
|
628
642
|
#: :class:`~eodag.plugins.authentication.token_exchange.OIDCTokenExchangeAuth`
|
|
629
643
|
#: Identifies the issuer of the `subject_token`
|
|
630
644
|
subject_issuer: str
|
|
645
|
+
#: :class:`~eodag.plugins.authentication.token.TokenAuth`
|
|
646
|
+
#: :class:`~eodag.plugins.authentication.openid_connect.OIDCRefreshTokenBase`
|
|
647
|
+
#: Safety buffer to prevent token rejection from unexpected expiry between validity check and request.
|
|
648
|
+
token_expiration_margin: int
|
|
631
649
|
#: :class:`~eodag.plugins.authentication.token_exchange.OIDCTokenExchangeAuth`
|
|
632
650
|
#: Audience that the token ID is intended for. :attr:`~eodag.config.PluginConfig.client_id` of the Relying Party
|
|
633
651
|
audience: str
|
|
@@ -648,6 +666,7 @@ class PluginConfig(yaml.YAMLObject):
|
|
|
648
666
|
@classmethod
|
|
649
667
|
def from_mapping(cls, mapping: dict[str, Any]) -> PluginConfig:
|
|
650
668
|
"""Build a :class:`~eodag.config.PluginConfig` from a mapping"""
|
|
669
|
+
cls.validate(tuple(mapping.keys()))
|
|
651
670
|
c = cls()
|
|
652
671
|
c.__dict__.update(mapping)
|
|
653
672
|
return c
|
|
@@ -657,7 +676,7 @@ class PluginConfig(yaml.YAMLObject):
|
|
|
657
676
|
"""Validate a :class:`~eodag.config.PluginConfig`"""
|
|
658
677
|
if "type" not in config_keys:
|
|
659
678
|
raise ValidationError(
|
|
660
|
-
"A Plugin config must specify the Plugin it configures"
|
|
679
|
+
"A Plugin config must specify the type of Plugin it configures"
|
|
661
680
|
)
|
|
662
681
|
|
|
663
682
|
def update(self, mapping: Optional[dict[Any, Any]]) -> None:
|
|
@@ -689,6 +708,8 @@ def load_default_config() -> dict[str, ProviderConfig]:
|
|
|
689
708
|
def load_config(config_path: str) -> dict[str, ProviderConfig]:
|
|
690
709
|
"""Load the providers configuration into a dictionary from a given file
|
|
691
710
|
|
|
711
|
+
If EODAG_PROVIDERS_WHITELIST is set, only load listed providers.
|
|
712
|
+
|
|
692
713
|
:param config_path: The path to the provider config file
|
|
693
714
|
:returns: The default provider's configuration
|
|
694
715
|
"""
|
|
@@ -701,10 +722,23 @@ def load_config(config_path: str) -> dict[str, ProviderConfig]:
|
|
|
701
722
|
except yaml.parser.ParserError as e:
|
|
702
723
|
logger.error("Unable to load configuration")
|
|
703
724
|
raise e
|
|
725
|
+
|
|
704
726
|
stac_provider_config = load_stac_provider_config()
|
|
727
|
+
|
|
728
|
+
whitelist_env = os.getenv("EODAG_PROVIDERS_WHITELIST")
|
|
729
|
+
whitelist = None
|
|
730
|
+
if whitelist_env:
|
|
731
|
+
whitelist = {provider for provider in whitelist_env.split(",")}
|
|
732
|
+
logger.info("Using providers whitelist: %s", ", ".join(whitelist))
|
|
733
|
+
|
|
705
734
|
for provider_config in providers_configs:
|
|
735
|
+
if provider_config is None or (
|
|
736
|
+
whitelist and provider_config.name not in whitelist
|
|
737
|
+
):
|
|
738
|
+
continue
|
|
706
739
|
provider_config_init(provider_config, stac_provider_config)
|
|
707
740
|
config[provider_config.name] = provider_config
|
|
741
|
+
|
|
708
742
|
return config
|
|
709
743
|
|
|
710
744
|
|
|
@@ -803,7 +837,9 @@ def provider_config_init(
|
|
|
803
837
|
pass
|
|
804
838
|
|
|
805
839
|
|
|
806
|
-
def override_config_from_file(
|
|
840
|
+
def override_config_from_file(
|
|
841
|
+
config: dict[str, ProviderConfig], file_path: str
|
|
842
|
+
) -> None:
|
|
807
843
|
"""Override a configuration with the values in a file
|
|
808
844
|
|
|
809
845
|
:param config: An eodag providers configuration dictionary
|
|
@@ -818,10 +854,11 @@ def override_config_from_file(config: dict[str, Any], file_path: str) -> None:
|
|
|
818
854
|
except yaml.parser.ParserError as e:
|
|
819
855
|
logger.error("Unable to load user configuration file")
|
|
820
856
|
raise e
|
|
857
|
+
|
|
821
858
|
override_config_from_mapping(config, config_in_file)
|
|
822
859
|
|
|
823
860
|
|
|
824
|
-
def override_config_from_env(config: dict[str,
|
|
861
|
+
def override_config_from_env(config: dict[str, ProviderConfig]) -> None:
|
|
825
862
|
"""Override a configuration with environment variables values
|
|
826
863
|
|
|
827
864
|
:param config: An eodag providers configuration dictionary
|
|
@@ -892,16 +929,25 @@ def override_config_from_env(config: dict[str, Any]) -> None:
|
|
|
892
929
|
|
|
893
930
|
|
|
894
931
|
def override_config_from_mapping(
|
|
895
|
-
config: dict[str,
|
|
932
|
+
config: dict[str, ProviderConfig], mapping: dict[str, Any]
|
|
896
933
|
) -> None:
|
|
897
|
-
"""Override a configuration with the values in a mapping
|
|
934
|
+
"""Override a configuration with the values in a mapping.
|
|
935
|
+
|
|
936
|
+
If the environment variable ``EODAG_PROVIDERS_WHITELIST`` is set (as a comma-separated list of provider names),
|
|
937
|
+
only the listed providers will be used from the mapping. All other providers in the mapping will be ignored.
|
|
898
938
|
|
|
899
939
|
:param config: An eodag providers configuration dictionary
|
|
900
940
|
:param mapping: The mapping containing the values to be overriden
|
|
901
941
|
"""
|
|
942
|
+
whitelist_env = os.getenv("EODAG_PROVIDERS_WHITELIST")
|
|
943
|
+
whitelist = None
|
|
944
|
+
if whitelist_env:
|
|
945
|
+
whitelist = {provider for provider in whitelist_env.split(",")}
|
|
946
|
+
|
|
902
947
|
for provider, new_conf in mapping.items():
|
|
903
948
|
# check if metada-mapping as already been built as jsonpath in providers_config
|
|
904
|
-
|
|
949
|
+
# or provider not in whitelist
|
|
950
|
+
if not isinstance(new_conf, dict) or (whitelist and provider not in whitelist):
|
|
905
951
|
continue
|
|
906
952
|
new_conf_search = new_conf.get("search", {}) or {}
|
|
907
953
|
new_conf_api = new_conf.get("api", {}) or {}
|
|
@@ -930,7 +976,7 @@ def override_config_from_mapping(
|
|
|
930
976
|
)
|
|
931
977
|
|
|
932
978
|
# try overriding conf
|
|
933
|
-
old_conf: Optional[
|
|
979
|
+
old_conf: Optional[ProviderConfig] = config.get(provider)
|
|
934
980
|
if old_conf is not None:
|
|
935
981
|
old_conf.update(new_conf)
|
|
936
982
|
else:
|
|
@@ -60,8 +60,10 @@ class KeycloakOIDCPasswordAuth(OIDCRefreshTokenBase):
|
|
|
60
60
|
The allowed audiences that have to be present in the user token.
|
|
61
61
|
* :attr:`~eodag.config.PluginConfig.auth_error_code` (``int``): which error code is
|
|
62
62
|
returned in case of an authentication error
|
|
63
|
-
* :attr:`~eodag.config.PluginConfig.ssl_verify` (``bool``):
|
|
63
|
+
* :attr:`~eodag.config.PluginConfig.ssl_verify` (``bool``): If the SSL certificates
|
|
64
64
|
should be verified in the token request; default: ``True``
|
|
65
|
+
* :attr:`~eodag.config.PluginConfig.token_expiration_margin` (``int``): The margin of time (in seconds)
|
|
66
|
+
before a token is considered expired. Default: 60 seconds
|
|
65
67
|
|
|
66
68
|
Using :class:`~eodag.plugins.download.http.HTTPDownload` a download link
|
|
67
69
|
``http://example.com?foo=bar`` will become
|
|
@@ -30,7 +30,14 @@ from lxml import etree
|
|
|
30
30
|
from requests.auth import AuthBase
|
|
31
31
|
|
|
32
32
|
from eodag.plugins.authentication import Authentication
|
|
33
|
-
from eodag.utils import
|
|
33
|
+
from eodag.utils import (
|
|
34
|
+
DEFAULT_TOKEN_EXPIRATION_MARGIN,
|
|
35
|
+
HTTP_REQ_TIMEOUT,
|
|
36
|
+
USER_AGENT,
|
|
37
|
+
parse_qs,
|
|
38
|
+
repeatfunc,
|
|
39
|
+
urlparse,
|
|
40
|
+
)
|
|
34
41
|
from eodag.utils.exceptions import (
|
|
35
42
|
AuthenticationError,
|
|
36
43
|
MisconfiguredError,
|
|
@@ -117,13 +124,22 @@ class OIDCRefreshTokenBase(Authentication):
|
|
|
117
124
|
|
|
118
125
|
def _get_access_token(self) -> str:
|
|
119
126
|
now = datetime.now(timezone.utc)
|
|
120
|
-
|
|
127
|
+
expiration_margin = timedelta(
|
|
128
|
+
seconds=getattr(
|
|
129
|
+
self.config, "token_expiration_margin", DEFAULT_TOKEN_EXPIRATION_MARGIN
|
|
130
|
+
)
|
|
131
|
+
)
|
|
132
|
+
|
|
133
|
+
if self.access_token and self.access_token_expiration - now > expiration_margin:
|
|
121
134
|
logger.debug(
|
|
122
135
|
f"Existing access_token is still valid until {self.access_token_expiration.isoformat()}."
|
|
123
136
|
)
|
|
124
137
|
return self.access_token
|
|
125
138
|
|
|
126
|
-
elif
|
|
139
|
+
elif (
|
|
140
|
+
self.refresh_token
|
|
141
|
+
and self.refresh_token_expiration - now > expiration_margin
|
|
142
|
+
):
|
|
127
143
|
response = self._get_token_with_refresh_token()
|
|
128
144
|
logger.debug(
|
|
129
145
|
"access_token expired, fetching new access_token using refresh_token"
|
|
@@ -268,6 +284,8 @@ class OIDCAuthorizationCodeFlowAuth(OIDCRefreshTokenBase):
|
|
|
268
284
|
Refers to the name of the query param to be used in the query request
|
|
269
285
|
* :attr:`~eodag.config.PluginConfig.refresh_token_key` (``str``): The key pointing to
|
|
270
286
|
the refresh_token in the json response to the POST request to the token server
|
|
287
|
+
* :attr:`~eodag.config.PluginConfig.token_expiration_margin` (``int``): The margin of time (in seconds)
|
|
288
|
+
before a token is considered expired. Default: 60 seconds.
|
|
271
289
|
|
|
272
290
|
"""
|
|
273
291
|
|
|
@@ -31,6 +31,7 @@ from urllib3 import Retry
|
|
|
31
31
|
|
|
32
32
|
from eodag.plugins.authentication.base import Authentication
|
|
33
33
|
from eodag.utils import (
|
|
34
|
+
DEFAULT_TOKEN_EXPIRATION_MARGIN,
|
|
34
35
|
HTTP_REQ_TIMEOUT,
|
|
35
36
|
REQ_RETRY_BACKOFF_FACTOR,
|
|
36
37
|
REQ_RETRY_STATUS_FORCELIST,
|
|
@@ -90,6 +91,8 @@ class TokenAuth(Authentication):
|
|
|
90
91
|
* :attr:`~eodag.config.PluginConfig.retry_status_forcelist` (``list[int]``): :class:`urllib3.util.Retry`
|
|
91
92
|
``status_forcelist`` parameter, list of integer HTTP status codes that we should force a retry on; default:
|
|
92
93
|
``[401, 429, 500, 502, 503, 504]``
|
|
94
|
+
* :attr:`~eodag.config.PluginConfig.token_expiration_margin` (``int``): The margin of time (in seconds) before
|
|
95
|
+
a token is considered expired. Default: 60 seconds.
|
|
93
96
|
"""
|
|
94
97
|
|
|
95
98
|
def __init__(self, provider: str, config: PluginConfig) -> None:
|
|
@@ -152,9 +155,18 @@ class TokenAuth(Authentication):
|
|
|
152
155
|
|
|
153
156
|
# Use a thread lock to avoid several threads requesting the token at the same time
|
|
154
157
|
with self.auth_lock:
|
|
155
|
-
|
|
158
|
+
expiration_margin = timedelta(
|
|
159
|
+
seconds=getattr(
|
|
160
|
+
self.config,
|
|
161
|
+
"token_expiration_margin",
|
|
162
|
+
DEFAULT_TOKEN_EXPIRATION_MARGIN,
|
|
163
|
+
)
|
|
164
|
+
)
|
|
156
165
|
self.validate_config_credentials()
|
|
157
|
-
if
|
|
166
|
+
if (
|
|
167
|
+
self.token
|
|
168
|
+
and self.token_expiration - datetime.now() > expiration_margin
|
|
169
|
+
):
|
|
158
170
|
logger.debug("using existing access token")
|
|
159
171
|
return RequestsTokenAuth(
|
|
160
172
|
self.token, "header", headers=getattr(self.config, "headers", {})
|
eodag/plugins/download/aws.py
CHANGED
|
@@ -207,7 +207,7 @@ class AwsDownload(Download):
|
|
|
207
207
|
:param config: Download plugin configuration:
|
|
208
208
|
|
|
209
209
|
* :attr:`~eodag.config.PluginConfig.type` (``str``) (**mandatory**): AwsDownload
|
|
210
|
-
* :attr:`~eodag.config.PluginConfig.
|
|
210
|
+
* :attr:`~eodag.config.PluginConfig.s3_endpoint` (``str``): s3 endpoint url
|
|
211
211
|
* :attr:`~eodag.config.PluginConfig.requester_pays` (``bool``): whether download is done
|
|
212
212
|
from a requester-pays bucket or not; default: ``False``
|
|
213
213
|
* :attr:`~eodag.config.PluginConfig.flatten_top_dirs` (``bool``): if the directory structure
|
eodag/plugins/manager.py
CHANGED
|
@@ -190,7 +190,7 @@ class PluginManager:
|
|
|
190
190
|
plugin = cast(Api, self._build_plugin(config.name, api, Api))
|
|
191
191
|
else:
|
|
192
192
|
raise MisconfiguredError(
|
|
193
|
-
f"No search plugin
|
|
193
|
+
f"No search plugin configured for {config.name}."
|
|
194
194
|
)
|
|
195
195
|
return plugin
|
|
196
196
|
|
eodag/plugins/search/base.py
CHANGED
|
@@ -275,7 +275,8 @@ class Search(PluginTopic):
|
|
|
275
275
|
if eodag_sort_order[:3] != "ASC" and eodag_sort_order[:3] != "DES":
|
|
276
276
|
raise ValidationError(
|
|
277
277
|
"Sorting order is invalid: it must be set to 'ASC' (ASCENDING) or "
|
|
278
|
-
f"'DESC' (DESCENDING), got '{eodag_sort_order}' with '{eodag_sort_param}' instead"
|
|
278
|
+
f"'DESC' (DESCENDING), got '{eodag_sort_order}' with '{eodag_sort_param}' instead",
|
|
279
|
+
{eodag_sort_param},
|
|
279
280
|
)
|
|
280
281
|
eodag_sort_order = eodag_sort_order[:3]
|
|
281
282
|
|
|
@@ -296,7 +297,7 @@ class Search(PluginTopic):
|
|
|
296
297
|
raise ValidationError(
|
|
297
298
|
f"'{eodag_sort_param}' parameter is called several times to sort results with different "
|
|
298
299
|
"sorting orders. Please set it to only one ('ASC' (ASCENDING) or 'DESC' (DESCENDING))",
|
|
299
|
-
|
|
300
|
+
{eodag_sort_param},
|
|
300
301
|
)
|
|
301
302
|
provider_sort_by_tuples_used.append(provider_sort_by_tuple)
|
|
302
303
|
|
|
@@ -722,7 +722,7 @@ class ECMWFSearch(PostJsonSearch):
|
|
|
722
722
|
"geometry",
|
|
723
723
|
}:
|
|
724
724
|
raise ValidationError(
|
|
725
|
-
f"{key} is not a queryable parameter for {self.provider}"
|
|
725
|
+
f"'{key}' is not a queryable parameter for {self.provider}", {key}
|
|
726
726
|
)
|
|
727
727
|
|
|
728
728
|
formated_filters[key] = v
|
|
@@ -781,7 +781,9 @@ class ECMWFSearch(PostJsonSearch):
|
|
|
781
781
|
and keyword.removeprefix(ECMWF_PREFIX)
|
|
782
782
|
not in set(list(available_values.keys()) + [f["name"] for f in form])
|
|
783
783
|
):
|
|
784
|
-
raise ValidationError(
|
|
784
|
+
raise ValidationError(
|
|
785
|
+
f"'{keyword}' is not a queryable parameter", {keyword}
|
|
786
|
+
)
|
|
785
787
|
|
|
786
788
|
# generate queryables
|
|
787
789
|
if form:
|
|
@@ -868,7 +870,8 @@ class ECMWFSearch(PostJsonSearch):
|
|
|
868
870
|
# we only compare list of strings.
|
|
869
871
|
if isinstance(values, dict):
|
|
870
872
|
raise ValidationError(
|
|
871
|
-
f"Parameter value as object is not supported: {keyword}={values}"
|
|
873
|
+
f"Parameter value as object is not supported: {keyword}={values}",
|
|
874
|
+
{keyword},
|
|
872
875
|
)
|
|
873
876
|
|
|
874
877
|
# We convert every single value to a list of string
|
|
@@ -934,7 +937,10 @@ class ECMWFSearch(PostJsonSearch):
|
|
|
934
937
|
raise ValidationError(
|
|
935
938
|
f"{keyword}={values} is not available"
|
|
936
939
|
f"{all_keywords_str}."
|
|
937
|
-
f" Allowed values are {', '.join(allowed_values)}."
|
|
940
|
+
f" Allowed values are {', '.join(allowed_values)}.",
|
|
941
|
+
set(
|
|
942
|
+
[keyword] + [k for k in parsed_keywords if k in input_keywords]
|
|
943
|
+
),
|
|
938
944
|
)
|
|
939
945
|
|
|
940
946
|
parsed_keywords.append(keyword)
|
|
@@ -32,6 +32,7 @@ from eodag.api.product.metadata_mapping import (
|
|
|
32
32
|
)
|
|
33
33
|
from eodag.plugins.search import PreparedSearch
|
|
34
34
|
from eodag.plugins.search.base import Search
|
|
35
|
+
from eodag.types.queryables import Queryables
|
|
35
36
|
from eodag.utils import (
|
|
36
37
|
DEFAULT_ITEMS_PER_PAGE,
|
|
37
38
|
DEFAULT_MISSION_START_DATE,
|
|
@@ -359,7 +360,19 @@ class DataRequestSearch(Search):
|
|
|
359
360
|
ssl_verify = getattr(self.config.ssl_verify, "ssl_verify", True)
|
|
360
361
|
try:
|
|
361
362
|
url = self.config.data_request_url
|
|
362
|
-
|
|
363
|
+
try:
|
|
364
|
+
request_body = format_query_params(
|
|
365
|
+
eodag_product_type, self.config, kwargs
|
|
366
|
+
)
|
|
367
|
+
except ValidationError as err:
|
|
368
|
+
not_queryable_search_param = Queryables.get_queryable_from_alias(
|
|
369
|
+
str(err.message).split(":")[-1].strip()
|
|
370
|
+
)
|
|
371
|
+
raise ValidationError(
|
|
372
|
+
f"Search parameters which are not queryable are disallowed for {product_type} on "
|
|
373
|
+
f"{self.provider}: please remove '{not_queryable_search_param}' from your search parameters",
|
|
374
|
+
{not_queryable_search_param},
|
|
375
|
+
) from err
|
|
363
376
|
logger.debug(
|
|
364
377
|
f"Sending search job request to {url} with {str(request_body)}"
|
|
365
378
|
)
|
eodag/plugins/search/qssearch.py
CHANGED
|
@@ -811,7 +811,10 @@ class QueryStringSearch(Search):
|
|
|
811
811
|
) -> tuple[dict[str, Any], str]:
|
|
812
812
|
"""Build The query string using the search parameters"""
|
|
813
813
|
logger.debug("Building the query string that will be used for search")
|
|
814
|
-
|
|
814
|
+
error_context = f"Product type: {product_type} / provider : {self.provider}"
|
|
815
|
+
query_params = format_query_params(
|
|
816
|
+
product_type, self.config, query_dict, error_context
|
|
817
|
+
)
|
|
815
818
|
|
|
816
819
|
# Build the final query string, in one go without quoting it
|
|
817
820
|
# (some providers do not operate well with urlencoded and quoted query strings)
|
|
@@ -1781,7 +1784,10 @@ class StacSearch(PostJsonSearch):
|
|
|
1781
1784
|
query_dict.setdefault("startTimeFromAscendingNode", "..")
|
|
1782
1785
|
query_dict.setdefault("completionTimeFromAscendingNode", "..")
|
|
1783
1786
|
|
|
1784
|
-
|
|
1787
|
+
error_context = f"Product type: {product_type} / provider : {self.provider}"
|
|
1788
|
+
query_params = format_query_params(
|
|
1789
|
+
product_type, self.config, query_dict, error_context
|
|
1790
|
+
)
|
|
1785
1791
|
|
|
1786
1792
|
# Build the final query string, in one go without quoting it
|
|
1787
1793
|
# (some providers do not operate well with urlencoded and quoted query strings)
|