eodag 3.4.3__py3-none-any.whl → 3.5.1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- eodag/api/core.py +90 -27
- eodag/api/product/_product.py +33 -0
- eodag/api/product/metadata_mapping.py +31 -2
- eodag/config.py +56 -10
- eodag/plugins/authentication/keycloak.py +3 -1
- eodag/plugins/authentication/openid_connect.py +21 -3
- eodag/plugins/authentication/token.py +14 -2
- eodag/plugins/authentication/token_exchange.py +1 -0
- eodag/plugins/download/aws.py +1 -1
- eodag/plugins/manager.py +1 -1
- eodag/plugins/search/base.py +3 -2
- eodag/plugins/search/build_search_result.py +13 -6
- eodag/plugins/search/data_request_search.py +14 -1
- eodag/plugins/search/qssearch.py +8 -2
- eodag/resources/ext_product_types.json +1 -1
- eodag/resources/providers.yml +4 -0
- eodag/rest/core.py +3 -1
- eodag/rest/errors.py +9 -4
- eodag/rest/server.py +3 -1
- eodag/rest/stac.py +22 -1
- eodag/utils/__init__.py +3 -0
- eodag/utils/cache.py +68 -0
- eodag/utils/env.py +29 -0
- {eodag-3.4.3.dist-info → eodag-3.5.1.dist-info}/METADATA +3 -3
- {eodag-3.4.3.dist-info → eodag-3.5.1.dist-info}/RECORD +29 -27
- {eodag-3.4.3.dist-info → eodag-3.5.1.dist-info}/WHEEL +0 -0
- {eodag-3.4.3.dist-info → eodag-3.5.1.dist-info}/entry_points.txt +0 -0
- {eodag-3.4.3.dist-info → eodag-3.5.1.dist-info}/licenses/LICENSE +0 -0
- {eodag-3.4.3.dist-info → eodag-3.5.1.dist-info}/top_level.txt +0 -0
eodag/api/core.py
CHANGED
|
@@ -36,7 +36,7 @@ from whoosh.index import exists_in, open_dir
|
|
|
36
36
|
from whoosh.qparser import QueryParser
|
|
37
37
|
|
|
38
38
|
from eodag.api.product.metadata_mapping import (
|
|
39
|
-
|
|
39
|
+
NOT_AVAILABLE,
|
|
40
40
|
mtd_cfg_as_conversion_and_querypath,
|
|
41
41
|
)
|
|
42
42
|
from eodag.api.search_result import SearchResult
|
|
@@ -79,6 +79,7 @@ from eodag.utils import (
|
|
|
79
79
|
string_to_jsonpath,
|
|
80
80
|
uri_to_path,
|
|
81
81
|
)
|
|
82
|
+
from eodag.utils.env import is_env_var_true
|
|
82
83
|
from eodag.utils.exceptions import (
|
|
83
84
|
AuthenticationError,
|
|
84
85
|
EodagError,
|
|
@@ -175,12 +176,19 @@ class EODataAccessGateway:
|
|
|
175
176
|
share_credentials(self.providers_config)
|
|
176
177
|
|
|
177
178
|
# init updated providers conf
|
|
179
|
+
strict_mode = is_env_var_true("EODAG_STRICT_PRODUCT_TYPES")
|
|
180
|
+
available_product_types = set(self.product_types_config.source.keys())
|
|
181
|
+
|
|
178
182
|
for provider in self.providers_config.keys():
|
|
179
183
|
provider_config_init(
|
|
180
184
|
self.providers_config[provider],
|
|
181
185
|
load_stac_provider_config(),
|
|
182
186
|
)
|
|
183
187
|
|
|
188
|
+
self._sync_provider_product_types(
|
|
189
|
+
provider, available_product_types, strict_mode
|
|
190
|
+
)
|
|
191
|
+
|
|
184
192
|
# re-build _plugins_manager using up-to-date providers_config
|
|
185
193
|
self._plugins_manager.rebuild(self.providers_config)
|
|
186
194
|
|
|
@@ -226,6 +234,61 @@ class EODataAccessGateway:
|
|
|
226
234
|
)
|
|
227
235
|
self.set_locations_conf(locations_conf_path)
|
|
228
236
|
|
|
237
|
+
def _sync_provider_product_types(
|
|
238
|
+
self,
|
|
239
|
+
provider: str,
|
|
240
|
+
available_product_types: set[str],
|
|
241
|
+
strict_mode: bool,
|
|
242
|
+
) -> None:
|
|
243
|
+
"""
|
|
244
|
+
Synchronize product types for a provider based on strict or permissive mode.
|
|
245
|
+
|
|
246
|
+
In strict mode, removes product types not in available_product_types.
|
|
247
|
+
In permissive mode, adds empty product type configs for missing types.
|
|
248
|
+
|
|
249
|
+
:param provider: The provider name whose product types should be synchronized.
|
|
250
|
+
:param available_product_types: The set of available product type IDs.
|
|
251
|
+
:param strict_mode: If True, remove unknown product types; if False, add empty configs for them.
|
|
252
|
+
:returns: None
|
|
253
|
+
"""
|
|
254
|
+
provider_products = self.providers_config[provider].products
|
|
255
|
+
products_to_remove: list[str] = []
|
|
256
|
+
products_to_add: list[str] = []
|
|
257
|
+
|
|
258
|
+
for product_id in provider_products:
|
|
259
|
+
if product_id == GENERIC_PRODUCT_TYPE:
|
|
260
|
+
continue
|
|
261
|
+
|
|
262
|
+
if product_id not in available_product_types:
|
|
263
|
+
if strict_mode:
|
|
264
|
+
products_to_remove.append(product_id)
|
|
265
|
+
continue
|
|
266
|
+
|
|
267
|
+
empty_product = {
|
|
268
|
+
"title": product_id,
|
|
269
|
+
"abstract": NOT_AVAILABLE,
|
|
270
|
+
}
|
|
271
|
+
self.product_types_config.source[
|
|
272
|
+
product_id
|
|
273
|
+
] = empty_product # will update available_product_types
|
|
274
|
+
products_to_add.append(product_id)
|
|
275
|
+
|
|
276
|
+
if products_to_add:
|
|
277
|
+
logger.debug(
|
|
278
|
+
"Product types permissive mode, %s added (provider %s)",
|
|
279
|
+
", ".join(products_to_add),
|
|
280
|
+
provider,
|
|
281
|
+
)
|
|
282
|
+
|
|
283
|
+
if products_to_remove:
|
|
284
|
+
logger.debug(
|
|
285
|
+
"Product types strict mode, ignoring %s (provider %s)",
|
|
286
|
+
", ".join(products_to_remove),
|
|
287
|
+
provider,
|
|
288
|
+
)
|
|
289
|
+
for id in products_to_remove:
|
|
290
|
+
del self.providers_config[provider].products[id]
|
|
291
|
+
|
|
229
292
|
def get_version(self) -> str:
|
|
230
293
|
"""Get eodag package version"""
|
|
231
294
|
return version("eodag")
|
|
@@ -608,11 +671,14 @@ class EODataAccessGateway:
|
|
|
608
671
|
for product_type_id in p.products: # type: ignore
|
|
609
672
|
if product_type_id == GENERIC_PRODUCT_TYPE:
|
|
610
673
|
continue
|
|
674
|
+
|
|
611
675
|
config = self.product_types_config[product_type_id]
|
|
612
676
|
config["_id"] = product_type_id
|
|
677
|
+
|
|
613
678
|
if "alias" in config:
|
|
614
679
|
product_type_id = config["alias"]
|
|
615
680
|
product_type = {"ID": product_type_id, **config}
|
|
681
|
+
|
|
616
682
|
if product_type not in product_types:
|
|
617
683
|
product_types.append(product_type)
|
|
618
684
|
|
|
@@ -620,11 +686,18 @@ class EODataAccessGateway:
|
|
|
620
686
|
return sorted(product_types, key=itemgetter("ID"))
|
|
621
687
|
|
|
622
688
|
def fetch_product_types_list(self, provider: Optional[str] = None) -> None:
|
|
623
|
-
"""Fetch product types list and update if needed
|
|
689
|
+
"""Fetch product types list and update if needed.
|
|
690
|
+
|
|
691
|
+
If strict mode is enabled (by setting the ``EODAG_STRICT_PRODUCT_TYPES`` environment variable
|
|
692
|
+
to a truthy value), this method will not fetch or update product types and will return immediately.
|
|
624
693
|
|
|
625
694
|
:param provider: The name of a provider or provider-group for which product types
|
|
626
695
|
list should be updated. Defaults to all providers (None value).
|
|
627
696
|
"""
|
|
697
|
+
strict_mode = is_env_var_true("EODAG_STRICT_PRODUCT_TYPES")
|
|
698
|
+
if strict_mode:
|
|
699
|
+
return
|
|
700
|
+
|
|
628
701
|
providers_to_fetch = list(self.providers_config.keys())
|
|
629
702
|
# check if some providers are grouped under a group name which is not a provider name
|
|
630
703
|
if provider is not None and provider not in self.providers_config:
|
|
@@ -1321,7 +1394,11 @@ class EODataAccessGateway:
|
|
|
1321
1394
|
prev_product = None
|
|
1322
1395
|
next_page_url = None
|
|
1323
1396
|
next_page_query_obj = None
|
|
1397
|
+
number_matched = None
|
|
1324
1398
|
while True:
|
|
1399
|
+
# if count is enabled, it will only be performed on 1st iteration
|
|
1400
|
+
if iteration == 2:
|
|
1401
|
+
kwargs["count"] = False
|
|
1325
1402
|
if iteration > 1 and next_page_url:
|
|
1326
1403
|
pagination_config["next_page_url_tpl"] = next_page_url
|
|
1327
1404
|
if iteration > 1 and next_page_query_obj:
|
|
@@ -1329,11 +1406,13 @@ class EODataAccessGateway:
|
|
|
1329
1406
|
logger.info("Iterate search over multiple pages: page #%s", iteration)
|
|
1330
1407
|
try:
|
|
1331
1408
|
# remove unwanted kwargs for _do_search
|
|
1332
|
-
kwargs.pop("count", None)
|
|
1333
1409
|
kwargs.pop("raise_errors", None)
|
|
1334
1410
|
search_result = self._do_search(
|
|
1335
|
-
search_plugin,
|
|
1411
|
+
search_plugin, raise_errors=True, **kwargs
|
|
1336
1412
|
)
|
|
1413
|
+
# if count is enabled, it will only be performed on 1st iteration
|
|
1414
|
+
if iteration == 1:
|
|
1415
|
+
number_matched = search_result.number_matched
|
|
1337
1416
|
except Exception:
|
|
1338
1417
|
logger.warning(
|
|
1339
1418
|
"error at retrieval of data from %s, for params: %s",
|
|
@@ -1388,6 +1467,8 @@ class EODataAccessGateway:
|
|
|
1388
1467
|
)
|
|
1389
1468
|
last_page_with_products = iteration - 1
|
|
1390
1469
|
break
|
|
1470
|
+
# use count got from 1st iteration
|
|
1471
|
+
search_result.number_matched = number_matched
|
|
1391
1472
|
yield search_result
|
|
1392
1473
|
prev_product = product
|
|
1393
1474
|
# Prevent a last search if the current one returned less than the
|
|
@@ -1475,6 +1556,9 @@ class EODataAccessGateway:
|
|
|
1475
1556
|
)
|
|
1476
1557
|
self.fetch_product_types_list()
|
|
1477
1558
|
|
|
1559
|
+
# remove unwanted count
|
|
1560
|
+
kwargs.pop("count", None)
|
|
1561
|
+
|
|
1478
1562
|
search_plugins, search_kwargs = self._prepare_search(
|
|
1479
1563
|
start=start, end=end, geom=geom, locations=locations, **kwargs
|
|
1480
1564
|
)
|
|
@@ -1497,6 +1581,7 @@ class EODataAccessGateway:
|
|
|
1497
1581
|
for page_results in self.search_iter_page_plugin(
|
|
1498
1582
|
items_per_page=itp,
|
|
1499
1583
|
search_plugin=search_plugin,
|
|
1584
|
+
count=False,
|
|
1500
1585
|
**search_kwargs,
|
|
1501
1586
|
):
|
|
1502
1587
|
all_results.data.extend(page_results.data)
|
|
@@ -1907,29 +1992,7 @@ class EODataAccessGateway:
|
|
|
1907
1992
|
logger.debug("product type %s not found", eo_product.product_type)
|
|
1908
1993
|
|
|
1909
1994
|
if eo_product.search_intersection is not None:
|
|
1910
|
-
|
|
1911
|
-
eo_product
|
|
1912
|
-
)
|
|
1913
|
-
if len(eo_product.assets) > 0:
|
|
1914
|
-
matching_url = next(iter(eo_product.assets.values()))["href"]
|
|
1915
|
-
elif eo_product.properties.get("storageStatus") != ONLINE_STATUS:
|
|
1916
|
-
matching_url = eo_product.properties.get(
|
|
1917
|
-
"orderLink"
|
|
1918
|
-
) or eo_product.properties.get("downloadLink")
|
|
1919
|
-
else:
|
|
1920
|
-
matching_url = eo_product.properties.get("downloadLink")
|
|
1921
|
-
|
|
1922
|
-
try:
|
|
1923
|
-
auth_plugin = next(
|
|
1924
|
-
self._plugins_manager.get_auth_plugins(
|
|
1925
|
-
search_plugin.provider,
|
|
1926
|
-
matching_url=matching_url,
|
|
1927
|
-
matching_conf=download_plugin.config,
|
|
1928
|
-
)
|
|
1929
|
-
)
|
|
1930
|
-
except StopIteration:
|
|
1931
|
-
auth_plugin = None
|
|
1932
|
-
eo_product.register_downloader(download_plugin, auth_plugin)
|
|
1995
|
+
eo_product._register_downloader_from_manager(self._plugins_manager)
|
|
1933
1996
|
|
|
1934
1997
|
results.extend(res)
|
|
1935
1998
|
total_results = (
|
eodag/api/product/_product.py
CHANGED
|
@@ -43,6 +43,7 @@ from eodag.api.product.metadata_mapping import (
|
|
|
43
43
|
DEFAULT_GEOMETRY,
|
|
44
44
|
NOT_AVAILABLE,
|
|
45
45
|
NOT_MAPPED,
|
|
46
|
+
ONLINE_STATUS,
|
|
46
47
|
)
|
|
47
48
|
from eodag.utils import (
|
|
48
49
|
DEFAULT_DOWNLOAD_TIMEOUT,
|
|
@@ -62,6 +63,7 @@ if TYPE_CHECKING:
|
|
|
62
63
|
from eodag.plugins.apis.base import Api
|
|
63
64
|
from eodag.plugins.authentication.base import Authentication
|
|
64
65
|
from eodag.plugins.download.base import Download
|
|
66
|
+
from eodag.plugins.manager import PluginManager
|
|
65
67
|
from eodag.types.download_args import DownloadConf
|
|
66
68
|
from eodag.utils import Unpack
|
|
67
69
|
|
|
@@ -238,6 +240,37 @@ class EOProduct:
|
|
|
238
240
|
f"Unable to get {e.args[0]} key from EOProduct.properties"
|
|
239
241
|
)
|
|
240
242
|
|
|
243
|
+
def _register_downloader_from_manager(self, plugins_manager: PluginManager) -> None:
|
|
244
|
+
"""Register the downloader and authenticator for this EOProduct using the
|
|
245
|
+
provided plugins manager.
|
|
246
|
+
This method is typically called after the EOProduct has been created and
|
|
247
|
+
before any download operation is performed.
|
|
248
|
+
|
|
249
|
+
:param plugins_manager: The plugins manager instance to use for retrieving
|
|
250
|
+
the download and authentication plugins.
|
|
251
|
+
"""
|
|
252
|
+
download_plugin = plugins_manager.get_download_plugin(self)
|
|
253
|
+
if len(self.assets) > 0:
|
|
254
|
+
matching_url = next(iter(self.assets.values()))["href"]
|
|
255
|
+
elif self.properties.get("storageStatus") != ONLINE_STATUS:
|
|
256
|
+
matching_url = self.properties.get("orderLink") or self.properties.get(
|
|
257
|
+
"downloadLink"
|
|
258
|
+
)
|
|
259
|
+
else:
|
|
260
|
+
matching_url = self.properties.get("downloadLink")
|
|
261
|
+
|
|
262
|
+
try:
|
|
263
|
+
auth_plugin = next(
|
|
264
|
+
plugins_manager.get_auth_plugins(
|
|
265
|
+
self.provider,
|
|
266
|
+
matching_url=matching_url,
|
|
267
|
+
matching_conf=download_plugin.config,
|
|
268
|
+
)
|
|
269
|
+
)
|
|
270
|
+
except StopIteration:
|
|
271
|
+
auth_plugin = None
|
|
272
|
+
self.register_downloader(download_plugin, auth_plugin)
|
|
273
|
+
|
|
241
274
|
def register_downloader(
|
|
242
275
|
self, downloader: Union[Api, Download], authenticator: Optional[Authentication]
|
|
243
276
|
) -> None:
|
|
@@ -54,6 +54,7 @@ from eodag.utils import (
|
|
|
54
54
|
string_to_jsonpath,
|
|
55
55
|
update_nested_dict,
|
|
56
56
|
)
|
|
57
|
+
from eodag.utils.exceptions import ValidationError
|
|
57
58
|
|
|
58
59
|
if TYPE_CHECKING:
|
|
59
60
|
from shapely.geometry.base import BaseGeometry
|
|
@@ -1286,7 +1287,10 @@ def mtd_cfg_as_conversion_and_querypath(
|
|
|
1286
1287
|
|
|
1287
1288
|
|
|
1288
1289
|
def format_query_params(
|
|
1289
|
-
product_type: str,
|
|
1290
|
+
product_type: str,
|
|
1291
|
+
config: PluginConfig,
|
|
1292
|
+
query_dict: dict[str, Any],
|
|
1293
|
+
error_context: str = "",
|
|
1290
1294
|
) -> dict[str, Any]:
|
|
1291
1295
|
"""format the search parameters to query parameters"""
|
|
1292
1296
|
if "raise_errors" in query_dict.keys():
|
|
@@ -1299,10 +1303,26 @@ def format_query_params(
|
|
|
1299
1303
|
**config.products.get(product_type, {}).get("metadata_mapping", {}),
|
|
1300
1304
|
)
|
|
1301
1305
|
|
|
1306
|
+
# Raise error if non-queryables parameters are used and raise_mtd_discovery_error configured
|
|
1307
|
+
if (
|
|
1308
|
+
raise_mtd_discovery_error := config.products.get(product_type, {})
|
|
1309
|
+
.get("discover_metadata", {})
|
|
1310
|
+
.get("raise_mtd_discovery_error")
|
|
1311
|
+
) is None:
|
|
1312
|
+
raise_mtd_discovery_error = getattr(config, "discover_metadata", {}).get(
|
|
1313
|
+
"raise_mtd_discovery_error", False
|
|
1314
|
+
)
|
|
1315
|
+
|
|
1302
1316
|
query_params: dict[str, Any] = {}
|
|
1303
1317
|
# Get all the search parameters that are recognised as queryables by the
|
|
1304
1318
|
# provider (they appear in the queryables dictionary)
|
|
1305
|
-
queryables = _get_queryables(
|
|
1319
|
+
queryables = _get_queryables(
|
|
1320
|
+
query_dict,
|
|
1321
|
+
config,
|
|
1322
|
+
product_type_metadata_mapping,
|
|
1323
|
+
raise_mtd_discovery_error,
|
|
1324
|
+
error_context,
|
|
1325
|
+
)
|
|
1306
1326
|
|
|
1307
1327
|
for eodag_search_key, provider_search_key in queryables.items():
|
|
1308
1328
|
user_input = query_dict[eodag_search_key]
|
|
@@ -1438,6 +1458,8 @@ def _get_queryables(
|
|
|
1438
1458
|
search_params: dict[str, Any],
|
|
1439
1459
|
config: PluginConfig,
|
|
1440
1460
|
metadata_mapping: dict[str, Any],
|
|
1461
|
+
raise_mtd_discovery_error: bool,
|
|
1462
|
+
error_context: str,
|
|
1441
1463
|
) -> dict[str, Any]:
|
|
1442
1464
|
"""Retrieve the metadata mappings that are query-able"""
|
|
1443
1465
|
logger.debug("Retrieving queryable metadata from metadata_mapping")
|
|
@@ -1445,6 +1467,13 @@ def _get_queryables(
|
|
|
1445
1467
|
for eodag_search_key, user_input in search_params.items():
|
|
1446
1468
|
if user_input is not None:
|
|
1447
1469
|
md_mapping = metadata_mapping.get(eodag_search_key, (None, NOT_MAPPED))
|
|
1470
|
+
# raise an error when a query param not allowed by the provider is found
|
|
1471
|
+
if not isinstance(md_mapping, list) and raise_mtd_discovery_error:
|
|
1472
|
+
raise ValidationError(
|
|
1473
|
+
"Search parameters which are not queryable are disallowed for this product type on this provider: "
|
|
1474
|
+
f"please remove '{eodag_search_key}' from your search parameters. {error_context}",
|
|
1475
|
+
{eodag_search_key},
|
|
1476
|
+
)
|
|
1448
1477
|
_, md_value = md_mapping
|
|
1449
1478
|
# query param from defined metadata_mapping
|
|
1450
1479
|
if md_mapping is not None and isinstance(md_mapping, list):
|
eodag/config.py
CHANGED
|
@@ -194,12 +194,24 @@ class ProviderConfig(yaml.YAMLObject):
|
|
|
194
194
|
},
|
|
195
195
|
)
|
|
196
196
|
for key in PLUGINS_TOPICS_KEYS:
|
|
197
|
-
current_value: Optional[
|
|
197
|
+
current_value: Optional[PluginConfig] = getattr(self, key, None)
|
|
198
198
|
mapping_value = mapping.get(key, {})
|
|
199
199
|
if current_value is not None:
|
|
200
200
|
current_value.update(mapping_value)
|
|
201
201
|
elif mapping_value:
|
|
202
|
-
|
|
202
|
+
try:
|
|
203
|
+
setattr(self, key, PluginConfig.from_mapping(mapping_value))
|
|
204
|
+
except ValidationError as e:
|
|
205
|
+
logger.warning(
|
|
206
|
+
(
|
|
207
|
+
"Could not add %s Plugin config to %s configuration: %s. "
|
|
208
|
+
"Try updating existing %s Plugin configs instead."
|
|
209
|
+
),
|
|
210
|
+
key,
|
|
211
|
+
self.name,
|
|
212
|
+
str(e),
|
|
213
|
+
", ".join([k for k in PLUGINS_TOPICS_KEYS if hasattr(self, k)]),
|
|
214
|
+
)
|
|
203
215
|
|
|
204
216
|
|
|
205
217
|
class PluginConfig(yaml.YAMLObject):
|
|
@@ -256,9 +268,11 @@ class PluginConfig(yaml.YAMLObject):
|
|
|
256
268
|
#: Metadata regex pattern used for discovery in search result properties
|
|
257
269
|
metadata_pattern: str
|
|
258
270
|
#: Configuration/template that will be used to query for a discovered parameter
|
|
259
|
-
search_param: str
|
|
271
|
+
search_param: str | dict[str, Any]
|
|
260
272
|
#: Path to the metadata in search result
|
|
261
273
|
metadata_path: str
|
|
274
|
+
#: Whether an error must be raised when using a search parameter which is not queryable or not
|
|
275
|
+
raise_mtd_discovery_error: bool
|
|
262
276
|
|
|
263
277
|
class DiscoverProductTypes(TypedDict, total=False):
|
|
264
278
|
"""Configuration for product types discovery"""
|
|
@@ -628,6 +642,10 @@ class PluginConfig(yaml.YAMLObject):
|
|
|
628
642
|
#: :class:`~eodag.plugins.authentication.token_exchange.OIDCTokenExchangeAuth`
|
|
629
643
|
#: Identifies the issuer of the `subject_token`
|
|
630
644
|
subject_issuer: str
|
|
645
|
+
#: :class:`~eodag.plugins.authentication.token.TokenAuth`
|
|
646
|
+
#: :class:`~eodag.plugins.authentication.openid_connect.OIDCRefreshTokenBase`
|
|
647
|
+
#: Safety buffer to prevent token rejection from unexpected expiry between validity check and request.
|
|
648
|
+
token_expiration_margin: int
|
|
631
649
|
#: :class:`~eodag.plugins.authentication.token_exchange.OIDCTokenExchangeAuth`
|
|
632
650
|
#: Audience that the token ID is intended for. :attr:`~eodag.config.PluginConfig.client_id` of the Relying Party
|
|
633
651
|
audience: str
|
|
@@ -648,6 +666,7 @@ class PluginConfig(yaml.YAMLObject):
|
|
|
648
666
|
@classmethod
|
|
649
667
|
def from_mapping(cls, mapping: dict[str, Any]) -> PluginConfig:
|
|
650
668
|
"""Build a :class:`~eodag.config.PluginConfig` from a mapping"""
|
|
669
|
+
cls.validate(tuple(mapping.keys()))
|
|
651
670
|
c = cls()
|
|
652
671
|
c.__dict__.update(mapping)
|
|
653
672
|
return c
|
|
@@ -657,7 +676,7 @@ class PluginConfig(yaml.YAMLObject):
|
|
|
657
676
|
"""Validate a :class:`~eodag.config.PluginConfig`"""
|
|
658
677
|
if "type" not in config_keys:
|
|
659
678
|
raise ValidationError(
|
|
660
|
-
"A Plugin config must specify the Plugin it configures"
|
|
679
|
+
"A Plugin config must specify the type of Plugin it configures"
|
|
661
680
|
)
|
|
662
681
|
|
|
663
682
|
def update(self, mapping: Optional[dict[Any, Any]]) -> None:
|
|
@@ -689,6 +708,8 @@ def load_default_config() -> dict[str, ProviderConfig]:
|
|
|
689
708
|
def load_config(config_path: str) -> dict[str, ProviderConfig]:
|
|
690
709
|
"""Load the providers configuration into a dictionary from a given file
|
|
691
710
|
|
|
711
|
+
If EODAG_PROVIDERS_WHITELIST is set, only load listed providers.
|
|
712
|
+
|
|
692
713
|
:param config_path: The path to the provider config file
|
|
693
714
|
:returns: The default provider's configuration
|
|
694
715
|
"""
|
|
@@ -701,10 +722,23 @@ def load_config(config_path: str) -> dict[str, ProviderConfig]:
|
|
|
701
722
|
except yaml.parser.ParserError as e:
|
|
702
723
|
logger.error("Unable to load configuration")
|
|
703
724
|
raise e
|
|
725
|
+
|
|
704
726
|
stac_provider_config = load_stac_provider_config()
|
|
727
|
+
|
|
728
|
+
whitelist_env = os.getenv("EODAG_PROVIDERS_WHITELIST")
|
|
729
|
+
whitelist = None
|
|
730
|
+
if whitelist_env:
|
|
731
|
+
whitelist = {provider for provider in whitelist_env.split(",")}
|
|
732
|
+
logger.info("Using providers whitelist: %s", ", ".join(whitelist))
|
|
733
|
+
|
|
705
734
|
for provider_config in providers_configs:
|
|
735
|
+
if provider_config is None or (
|
|
736
|
+
whitelist and provider_config.name not in whitelist
|
|
737
|
+
):
|
|
738
|
+
continue
|
|
706
739
|
provider_config_init(provider_config, stac_provider_config)
|
|
707
740
|
config[provider_config.name] = provider_config
|
|
741
|
+
|
|
708
742
|
return config
|
|
709
743
|
|
|
710
744
|
|
|
@@ -803,7 +837,9 @@ def provider_config_init(
|
|
|
803
837
|
pass
|
|
804
838
|
|
|
805
839
|
|
|
806
|
-
def override_config_from_file(
|
|
840
|
+
def override_config_from_file(
|
|
841
|
+
config: dict[str, ProviderConfig], file_path: str
|
|
842
|
+
) -> None:
|
|
807
843
|
"""Override a configuration with the values in a file
|
|
808
844
|
|
|
809
845
|
:param config: An eodag providers configuration dictionary
|
|
@@ -818,10 +854,11 @@ def override_config_from_file(config: dict[str, Any], file_path: str) -> None:
|
|
|
818
854
|
except yaml.parser.ParserError as e:
|
|
819
855
|
logger.error("Unable to load user configuration file")
|
|
820
856
|
raise e
|
|
857
|
+
|
|
821
858
|
override_config_from_mapping(config, config_in_file)
|
|
822
859
|
|
|
823
860
|
|
|
824
|
-
def override_config_from_env(config: dict[str,
|
|
861
|
+
def override_config_from_env(config: dict[str, ProviderConfig]) -> None:
|
|
825
862
|
"""Override a configuration with environment variables values
|
|
826
863
|
|
|
827
864
|
:param config: An eodag providers configuration dictionary
|
|
@@ -892,16 +929,25 @@ def override_config_from_env(config: dict[str, Any]) -> None:
|
|
|
892
929
|
|
|
893
930
|
|
|
894
931
|
def override_config_from_mapping(
|
|
895
|
-
config: dict[str,
|
|
932
|
+
config: dict[str, ProviderConfig], mapping: dict[str, Any]
|
|
896
933
|
) -> None:
|
|
897
|
-
"""Override a configuration with the values in a mapping
|
|
934
|
+
"""Override a configuration with the values in a mapping.
|
|
935
|
+
|
|
936
|
+
If the environment variable ``EODAG_PROVIDERS_WHITELIST`` is set (as a comma-separated list of provider names),
|
|
937
|
+
only the listed providers will be used from the mapping. All other providers in the mapping will be ignored.
|
|
898
938
|
|
|
899
939
|
:param config: An eodag providers configuration dictionary
|
|
900
940
|
:param mapping: The mapping containing the values to be overriden
|
|
901
941
|
"""
|
|
942
|
+
whitelist_env = os.getenv("EODAG_PROVIDERS_WHITELIST")
|
|
943
|
+
whitelist = None
|
|
944
|
+
if whitelist_env:
|
|
945
|
+
whitelist = {provider for provider in whitelist_env.split(",")}
|
|
946
|
+
|
|
902
947
|
for provider, new_conf in mapping.items():
|
|
903
948
|
# check if metada-mapping as already been built as jsonpath in providers_config
|
|
904
|
-
|
|
949
|
+
# or provider not in whitelist
|
|
950
|
+
if not isinstance(new_conf, dict) or (whitelist and provider not in whitelist):
|
|
905
951
|
continue
|
|
906
952
|
new_conf_search = new_conf.get("search", {}) or {}
|
|
907
953
|
new_conf_api = new_conf.get("api", {}) or {}
|
|
@@ -930,7 +976,7 @@ def override_config_from_mapping(
|
|
|
930
976
|
)
|
|
931
977
|
|
|
932
978
|
# try overriding conf
|
|
933
|
-
old_conf: Optional[
|
|
979
|
+
old_conf: Optional[ProviderConfig] = config.get(provider)
|
|
934
980
|
if old_conf is not None:
|
|
935
981
|
old_conf.update(new_conf)
|
|
936
982
|
else:
|
|
@@ -60,8 +60,10 @@ class KeycloakOIDCPasswordAuth(OIDCRefreshTokenBase):
|
|
|
60
60
|
The allowed audiences that have to be present in the user token.
|
|
61
61
|
* :attr:`~eodag.config.PluginConfig.auth_error_code` (``int``): which error code is
|
|
62
62
|
returned in case of an authentication error
|
|
63
|
-
* :attr:`~eodag.config.PluginConfig.ssl_verify` (``bool``):
|
|
63
|
+
* :attr:`~eodag.config.PluginConfig.ssl_verify` (``bool``): If the SSL certificates
|
|
64
64
|
should be verified in the token request; default: ``True``
|
|
65
|
+
* :attr:`~eodag.config.PluginConfig.token_expiration_margin` (``int``): The margin of time (in seconds)
|
|
66
|
+
before a token is considered expired. Default: 60 seconds
|
|
65
67
|
|
|
66
68
|
Using :class:`~eodag.plugins.download.http.HTTPDownload` a download link
|
|
67
69
|
``http://example.com?foo=bar`` will become
|
|
@@ -30,7 +30,14 @@ from lxml import etree
|
|
|
30
30
|
from requests.auth import AuthBase
|
|
31
31
|
|
|
32
32
|
from eodag.plugins.authentication import Authentication
|
|
33
|
-
from eodag.utils import
|
|
33
|
+
from eodag.utils import (
|
|
34
|
+
DEFAULT_TOKEN_EXPIRATION_MARGIN,
|
|
35
|
+
HTTP_REQ_TIMEOUT,
|
|
36
|
+
USER_AGENT,
|
|
37
|
+
parse_qs,
|
|
38
|
+
repeatfunc,
|
|
39
|
+
urlparse,
|
|
40
|
+
)
|
|
34
41
|
from eodag.utils.exceptions import (
|
|
35
42
|
AuthenticationError,
|
|
36
43
|
MisconfiguredError,
|
|
@@ -117,13 +124,22 @@ class OIDCRefreshTokenBase(Authentication):
|
|
|
117
124
|
|
|
118
125
|
def _get_access_token(self) -> str:
|
|
119
126
|
now = datetime.now(timezone.utc)
|
|
120
|
-
|
|
127
|
+
expiration_margin = timedelta(
|
|
128
|
+
seconds=getattr(
|
|
129
|
+
self.config, "token_expiration_margin", DEFAULT_TOKEN_EXPIRATION_MARGIN
|
|
130
|
+
)
|
|
131
|
+
)
|
|
132
|
+
|
|
133
|
+
if self.access_token and self.access_token_expiration - now > expiration_margin:
|
|
121
134
|
logger.debug(
|
|
122
135
|
f"Existing access_token is still valid until {self.access_token_expiration.isoformat()}."
|
|
123
136
|
)
|
|
124
137
|
return self.access_token
|
|
125
138
|
|
|
126
|
-
elif
|
|
139
|
+
elif (
|
|
140
|
+
self.refresh_token
|
|
141
|
+
and self.refresh_token_expiration - now > expiration_margin
|
|
142
|
+
):
|
|
127
143
|
response = self._get_token_with_refresh_token()
|
|
128
144
|
logger.debug(
|
|
129
145
|
"access_token expired, fetching new access_token using refresh_token"
|
|
@@ -268,6 +284,8 @@ class OIDCAuthorizationCodeFlowAuth(OIDCRefreshTokenBase):
|
|
|
268
284
|
Refers to the name of the query param to be used in the query request
|
|
269
285
|
* :attr:`~eodag.config.PluginConfig.refresh_token_key` (``str``): The key pointing to
|
|
270
286
|
the refresh_token in the json response to the POST request to the token server
|
|
287
|
+
* :attr:`~eodag.config.PluginConfig.token_expiration_margin` (``int``): The margin of time (in seconds)
|
|
288
|
+
before a token is considered expired. Default: 60 seconds.
|
|
271
289
|
|
|
272
290
|
"""
|
|
273
291
|
|
|
@@ -31,6 +31,7 @@ from urllib3 import Retry
|
|
|
31
31
|
|
|
32
32
|
from eodag.plugins.authentication.base import Authentication
|
|
33
33
|
from eodag.utils import (
|
|
34
|
+
DEFAULT_TOKEN_EXPIRATION_MARGIN,
|
|
34
35
|
HTTP_REQ_TIMEOUT,
|
|
35
36
|
REQ_RETRY_BACKOFF_FACTOR,
|
|
36
37
|
REQ_RETRY_STATUS_FORCELIST,
|
|
@@ -90,6 +91,8 @@ class TokenAuth(Authentication):
|
|
|
90
91
|
* :attr:`~eodag.config.PluginConfig.retry_status_forcelist` (``list[int]``): :class:`urllib3.util.Retry`
|
|
91
92
|
``status_forcelist`` parameter, list of integer HTTP status codes that we should force a retry on; default:
|
|
92
93
|
``[401, 429, 500, 502, 503, 504]``
|
|
94
|
+
* :attr:`~eodag.config.PluginConfig.token_expiration_margin` (``int``): The margin of time (in seconds) before
|
|
95
|
+
a token is considered expired. Default: 60 seconds.
|
|
93
96
|
"""
|
|
94
97
|
|
|
95
98
|
def __init__(self, provider: str, config: PluginConfig) -> None:
|
|
@@ -152,9 +155,18 @@ class TokenAuth(Authentication):
|
|
|
152
155
|
|
|
153
156
|
# Use a thread lock to avoid several threads requesting the token at the same time
|
|
154
157
|
with self.auth_lock:
|
|
155
|
-
|
|
158
|
+
expiration_margin = timedelta(
|
|
159
|
+
seconds=getattr(
|
|
160
|
+
self.config,
|
|
161
|
+
"token_expiration_margin",
|
|
162
|
+
DEFAULT_TOKEN_EXPIRATION_MARGIN,
|
|
163
|
+
)
|
|
164
|
+
)
|
|
156
165
|
self.validate_config_credentials()
|
|
157
|
-
if
|
|
166
|
+
if (
|
|
167
|
+
self.token
|
|
168
|
+
and self.token_expiration - datetime.now() > expiration_margin
|
|
169
|
+
):
|
|
158
170
|
logger.debug("using existing access token")
|
|
159
171
|
return RequestsTokenAuth(
|
|
160
172
|
self.token, "header", headers=getattr(self.config, "headers", {})
|
eodag/plugins/download/aws.py
CHANGED
|
@@ -207,7 +207,7 @@ class AwsDownload(Download):
|
|
|
207
207
|
:param config: Download plugin configuration:
|
|
208
208
|
|
|
209
209
|
* :attr:`~eodag.config.PluginConfig.type` (``str``) (**mandatory**): AwsDownload
|
|
210
|
-
* :attr:`~eodag.config.PluginConfig.
|
|
210
|
+
* :attr:`~eodag.config.PluginConfig.s3_endpoint` (``str``): s3 endpoint url
|
|
211
211
|
* :attr:`~eodag.config.PluginConfig.requester_pays` (``bool``): whether download is done
|
|
212
212
|
from a requester-pays bucket or not; default: ``False``
|
|
213
213
|
* :attr:`~eodag.config.PluginConfig.flatten_top_dirs` (``bool``): if the directory structure
|
eodag/plugins/manager.py
CHANGED
|
@@ -190,7 +190,7 @@ class PluginManager:
|
|
|
190
190
|
plugin = cast(Api, self._build_plugin(config.name, api, Api))
|
|
191
191
|
else:
|
|
192
192
|
raise MisconfiguredError(
|
|
193
|
-
f"No search plugin
|
|
193
|
+
f"No search plugin configured for {config.name}."
|
|
194
194
|
)
|
|
195
195
|
return plugin
|
|
196
196
|
|
eodag/plugins/search/base.py
CHANGED
|
@@ -275,7 +275,8 @@ class Search(PluginTopic):
|
|
|
275
275
|
if eodag_sort_order[:3] != "ASC" and eodag_sort_order[:3] != "DES":
|
|
276
276
|
raise ValidationError(
|
|
277
277
|
"Sorting order is invalid: it must be set to 'ASC' (ASCENDING) or "
|
|
278
|
-
f"'DESC' (DESCENDING), got '{eodag_sort_order}' with '{eodag_sort_param}' instead"
|
|
278
|
+
f"'DESC' (DESCENDING), got '{eodag_sort_order}' with '{eodag_sort_param}' instead",
|
|
279
|
+
{eodag_sort_param},
|
|
279
280
|
)
|
|
280
281
|
eodag_sort_order = eodag_sort_order[:3]
|
|
281
282
|
|
|
@@ -296,7 +297,7 @@ class Search(PluginTopic):
|
|
|
296
297
|
raise ValidationError(
|
|
297
298
|
f"'{eodag_sort_param}' parameter is called several times to sort results with different "
|
|
298
299
|
"sorting orders. Please set it to only one ('ASC' (ASCENDING) or 'DESC' (DESCENDING))",
|
|
299
|
-
|
|
300
|
+
{eodag_sort_param},
|
|
300
301
|
)
|
|
301
302
|
provider_sort_by_tuples_used.append(provider_sort_by_tuple)
|
|
302
303
|
|