eodag 3.4.2__py3-none-any.whl → 3.4.3__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- eodag/api/core.py +12 -11
- eodag/api/product/_product.py +6 -6
- eodag/api/product/metadata_mapping.py +6 -6
- eodag/config.py +2 -2
- eodag/plugins/apis/ecmwf.py +5 -7
- eodag/plugins/apis/usgs.py +1 -1
- eodag/plugins/authentication/generic.py +5 -1
- eodag/plugins/authentication/openid_connect.py +68 -30
- eodag/plugins/authentication/sas_auth.py +1 -1
- eodag/plugins/crunch/filter_date.py +4 -8
- eodag/plugins/crunch/filter_property.py +1 -1
- eodag/plugins/download/aws.py +3 -3
- eodag/plugins/download/base.py +14 -7
- eodag/plugins/download/http.py +18 -12
- eodag/plugins/download/s3rest.py +9 -5
- eodag/plugins/search/base.py +3 -3
- eodag/plugins/search/build_search_result.py +4 -5
- eodag/plugins/search/data_request_search.py +4 -4
- eodag/plugins/search/qssearch.py +22 -29
- eodag/resources/ext_product_types.json +1 -1
- eodag/resources/providers.yml +2 -1
- eodag/rest/server.py +3 -3
- eodag/rest/stac.py +2 -2
- eodag/rest/types/queryables.py +2 -2
- eodag/types/__init__.py +27 -14
- eodag/types/queryables.py +23 -5
- eodag/utils/__init__.py +1 -1
- {eodag-3.4.2.dist-info → eodag-3.4.3.dist-info}/METADATA +2 -2
- {eodag-3.4.2.dist-info → eodag-3.4.3.dist-info}/RECORD +33 -33
- {eodag-3.4.2.dist-info → eodag-3.4.3.dist-info}/WHEEL +1 -1
- {eodag-3.4.2.dist-info → eodag-3.4.3.dist-info}/entry_points.txt +0 -0
- {eodag-3.4.2.dist-info → eodag-3.4.3.dist-info}/licenses/LICENSE +0 -0
- {eodag-3.4.2.dist-info → eodag-3.4.3.dist-info}/top_level.txt +0 -0
eodag/api/core.py
CHANGED
|
@@ -658,7 +658,7 @@ class EODataAccessGateway:
|
|
|
658
658
|
discovery_conf = getattr(
|
|
659
659
|
provider_search_config, "discover_product_types", {}
|
|
660
660
|
)
|
|
661
|
-
if discovery_conf.get("fetch_url"
|
|
661
|
+
if discovery_conf.get("fetch_url"):
|
|
662
662
|
providers_discovery_configs_fetchable[
|
|
663
663
|
provider_to_fetch
|
|
664
664
|
] = discovery_conf
|
|
@@ -744,7 +744,7 @@ class EODataAccessGateway:
|
|
|
744
744
|
user_discovery_conf == default_discovery_conf
|
|
745
745
|
or user_discovery_conf == default_discovery_conf_parsed
|
|
746
746
|
) and (
|
|
747
|
-
not default_discovery_conf.get("fetch_url"
|
|
747
|
+
not default_discovery_conf.get("fetch_url")
|
|
748
748
|
or "ext_product_types_conf" not in locals()
|
|
749
749
|
or "ext_product_types_conf" in locals()
|
|
750
750
|
and (
|
|
@@ -856,7 +856,7 @@ class EODataAccessGateway:
|
|
|
856
856
|
continue
|
|
857
857
|
if not getattr(
|
|
858
858
|
search_plugin_config, "discover_product_types", {}
|
|
859
|
-
).get("fetch_url"
|
|
859
|
+
).get("fetch_url"):
|
|
860
860
|
# conf has been updated and provider product types are no more discoverable
|
|
861
861
|
continue
|
|
862
862
|
provider_products_config = (
|
|
@@ -980,7 +980,7 @@ class EODataAccessGateway:
|
|
|
980
980
|
product_types = [
|
|
981
981
|
k
|
|
982
982
|
for k, v in self.product_types_config.items()
|
|
983
|
-
if v.get("alias"
|
|
983
|
+
if v.get("alias") == alias_or_id
|
|
984
984
|
]
|
|
985
985
|
|
|
986
986
|
if len(product_types) > 1:
|
|
@@ -1310,8 +1310,8 @@ class EODataAccessGateway:
|
|
|
1310
1310
|
# since it might be modified if the next_page_url mechanism is used by the
|
|
1311
1311
|
# plugin. (same thing for next_page_query_obj, next_page_query_obj with POST reqs)
|
|
1312
1312
|
pagination_config = getattr(search_plugin.config, "pagination", {})
|
|
1313
|
-
prev_next_page_url_tpl = pagination_config.get("next_page_url_tpl"
|
|
1314
|
-
prev_next_page_query_obj = pagination_config.get("next_page_query_obj"
|
|
1313
|
+
prev_next_page_url_tpl = pagination_config.get("next_page_url_tpl")
|
|
1314
|
+
prev_next_page_query_obj = pagination_config.get("next_page_query_obj")
|
|
1315
1315
|
# Page has to be set to a value even if use_next is True, this is required
|
|
1316
1316
|
# internally by the search plugin (see collect_search_urls)
|
|
1317
1317
|
kwargs.update(
|
|
@@ -1549,7 +1549,7 @@ class EODataAccessGateway:
|
|
|
1549
1549
|
:param kwargs: Search criteria to help finding the right product
|
|
1550
1550
|
:returns: A search result with one EO product or None at all
|
|
1551
1551
|
"""
|
|
1552
|
-
product_type = kwargs.get("productType"
|
|
1552
|
+
product_type = kwargs.get("productType")
|
|
1553
1553
|
if product_type is not None:
|
|
1554
1554
|
try:
|
|
1555
1555
|
product_type = self.get_product_type_from_alias(product_type)
|
|
@@ -1686,7 +1686,7 @@ class EODataAccessGateway:
|
|
|
1686
1686
|
* other criteria compatible with the provider
|
|
1687
1687
|
:returns: Search plugins list and the prepared kwargs to make a query.
|
|
1688
1688
|
"""
|
|
1689
|
-
product_type = kwargs.get("productType"
|
|
1689
|
+
product_type: Optional[str] = kwargs.get("productType")
|
|
1690
1690
|
if product_type is None:
|
|
1691
1691
|
try:
|
|
1692
1692
|
guesses = self.guess_product_type(**kwargs)
|
|
@@ -1706,7 +1706,7 @@ class EODataAccessGateway:
|
|
|
1706
1706
|
# By now, only use the best bet
|
|
1707
1707
|
product_type = guesses[0]
|
|
1708
1708
|
except NoMatchingProductType:
|
|
1709
|
-
queried_id = kwargs.get("id"
|
|
1709
|
+
queried_id = kwargs.get("id")
|
|
1710
1710
|
if queried_id is None:
|
|
1711
1711
|
logger.info(
|
|
1712
1712
|
"No product type could be guessed with provided arguments"
|
|
@@ -1750,7 +1750,7 @@ class EODataAccessGateway:
|
|
|
1750
1750
|
product_type
|
|
1751
1751
|
not in self._plugins_manager.product_type_to_provider_config_map.keys()
|
|
1752
1752
|
):
|
|
1753
|
-
if provider:
|
|
1753
|
+
if provider and product_type:
|
|
1754
1754
|
# Try to get specific product type from external provider
|
|
1755
1755
|
logger.debug(f"Fetching {provider} to find {product_type} product type")
|
|
1756
1756
|
self._fetch_external_product_type(provider, product_type)
|
|
@@ -1795,7 +1795,8 @@ class EODataAccessGateway:
|
|
|
1795
1795
|
# Add product_types_config to plugin config. This dict contains product
|
|
1796
1796
|
# type metadata that will also be stored in each product's properties.
|
|
1797
1797
|
for search_plugin in search_plugins:
|
|
1798
|
-
|
|
1798
|
+
if product_type is not None:
|
|
1799
|
+
self._attach_product_type_config(search_plugin, product_type)
|
|
1799
1800
|
|
|
1800
1801
|
return search_plugins, kwargs
|
|
1801
1802
|
|
eodag/api/product/_product.py
CHANGED
|
@@ -143,7 +143,7 @@ class EOProduct:
|
|
|
143
143
|
and "defaultGeometry" not in properties
|
|
144
144
|
):
|
|
145
145
|
raise MisconfiguredError(
|
|
146
|
-
f"No geometry available to build EOProduct(id={properties.get('id'
|
|
146
|
+
f"No geometry available to build EOProduct(id={properties.get('id')}, provider={provider})"
|
|
147
147
|
)
|
|
148
148
|
elif not properties["geometry"] or properties["geometry"] == NOT_AVAILABLE:
|
|
149
149
|
product_geometry = properties.pop("defaultGeometry", DEFAULT_GEOMETRY)
|
|
@@ -445,7 +445,7 @@ class EOProduct:
|
|
|
445
445
|
}
|
|
446
446
|
)
|
|
447
447
|
|
|
448
|
-
if self.properties.get("quicklook"
|
|
448
|
+
if self.properties.get("quicklook") is None:
|
|
449
449
|
logger.warning(
|
|
450
450
|
"Missing information to retrieve quicklook for EO product: %s",
|
|
451
451
|
self.properties["id"],
|
|
@@ -555,7 +555,7 @@ class EOProduct:
|
|
|
555
555
|
return NoDriver()
|
|
556
556
|
|
|
557
557
|
def _repr_html_(self):
|
|
558
|
-
thumbnail = self.properties.get("thumbnail"
|
|
558
|
+
thumbnail = self.properties.get("thumbnail")
|
|
559
559
|
thumbnail_html = (
|
|
560
560
|
f"<img src='{thumbnail}' width=100 alt='thumbnail'/>"
|
|
561
561
|
if thumbnail and not thumbnail.startswith("s3")
|
|
@@ -576,12 +576,12 @@ class EOProduct:
|
|
|
576
576
|
{dict_to_html_table({
|
|
577
577
|
"provider": self.provider,
|
|
578
578
|
"product_type": self.product_type,
|
|
579
|
-
"properties["id"]": self.properties.get('id'
|
|
579
|
+
"properties["id"]": self.properties.get('id'),
|
|
580
580
|
"properties["startTimeFromAscendingNode"]": self.properties.get(
|
|
581
|
-
'startTimeFromAscendingNode'
|
|
581
|
+
'startTimeFromAscendingNode'
|
|
582
582
|
),
|
|
583
583
|
"properties["completionTimeFromAscendingNode"]": self.properties.get(
|
|
584
|
-
'completionTimeFromAscendingNode'
|
|
584
|
+
'completionTimeFromAscendingNode'
|
|
585
585
|
),
|
|
586
586
|
}, brackets=False)}
|
|
587
587
|
<details><summary style='color: grey; margin-top: 10px;'>properties: ({len(
|
|
@@ -424,7 +424,7 @@ def format_metadata(search_param: str, *args: Any, **kwargs: Any) -> str:
|
|
|
424
424
|
return Polygon(polygon_args)
|
|
425
425
|
elif len(georss) == 1 and "multisurface" in georss[0].tag.lower():
|
|
426
426
|
# Multipolygon
|
|
427
|
-
from_proj = getattr(georss[0], "attrib", {}).get("srsName"
|
|
427
|
+
from_proj = getattr(georss[0], "attrib", {}).get("srsName")
|
|
428
428
|
if from_proj:
|
|
429
429
|
from_proj = pyproj.CRS(from_proj)
|
|
430
430
|
to_proj = pyproj.CRS(DEFAULT_PROJ)
|
|
@@ -1030,8 +1030,8 @@ def properties_from_json(
|
|
|
1030
1030
|
if not discovery_config:
|
|
1031
1031
|
discovery_config = {}
|
|
1032
1032
|
|
|
1033
|
-
discovery_pattern = discovery_config.get("metadata_pattern"
|
|
1034
|
-
discovery_path = discovery_config.get("metadata_path"
|
|
1033
|
+
discovery_pattern = discovery_config.get("metadata_pattern")
|
|
1034
|
+
discovery_path = discovery_config.get("metadata_path")
|
|
1035
1035
|
if discovery_pattern and discovery_path:
|
|
1036
1036
|
discovery_jsonpath = string_to_jsonpath(discovery_path)
|
|
1037
1037
|
discovered_properties = (
|
|
@@ -1215,8 +1215,8 @@ def properties_from_xml(
|
|
|
1215
1215
|
# adds missing discovered properties
|
|
1216
1216
|
if not discovery_config:
|
|
1217
1217
|
discovery_config = {}
|
|
1218
|
-
discovery_pattern = discovery_config.get("metadata_pattern"
|
|
1219
|
-
discovery_path = discovery_config.get("metadata_path"
|
|
1218
|
+
discovery_pattern = discovery_config.get("metadata_pattern")
|
|
1219
|
+
discovery_path = discovery_config.get("metadata_path")
|
|
1220
1220
|
if discovery_pattern and discovery_path:
|
|
1221
1221
|
discovered_properties = root.xpath(
|
|
1222
1222
|
discovery_path,
|
|
@@ -1524,7 +1524,7 @@ def get_provider_queryable_path(
|
|
|
1524
1524
|
:param metadata_mapping: metadata-mapping configuration
|
|
1525
1525
|
:returns: EODAG configured queryable path or None
|
|
1526
1526
|
"""
|
|
1527
|
-
parameter_conf = metadata_mapping.get(queryable
|
|
1527
|
+
parameter_conf = metadata_mapping.get(queryable)
|
|
1528
1528
|
if isinstance(parameter_conf, list):
|
|
1529
1529
|
return parameter_conf[0]
|
|
1530
1530
|
else:
|
eodag/config.py
CHANGED
|
@@ -851,7 +851,7 @@ def override_config_from_env(config: dict[str, Any]) -> None:
|
|
|
851
851
|
iter_parts = iter(parts)
|
|
852
852
|
env_type = get_type_hints(PluginConfig).get(next(iter_parts, ""), str)
|
|
853
853
|
child_env_type = (
|
|
854
|
-
get_type_hints(env_type).get(next(iter_parts, "")
|
|
854
|
+
get_type_hints(env_type).get(next(iter_parts, ""))
|
|
855
855
|
if isclass(env_type)
|
|
856
856
|
else None
|
|
857
857
|
)
|
|
@@ -961,7 +961,7 @@ def merge_configs(config: dict[str, Any], other_config: dict[str, Any]) -> None:
|
|
|
961
961
|
other_config = dict(config, **other_config)
|
|
962
962
|
|
|
963
963
|
for provider, new_conf in other_config.items():
|
|
964
|
-
old_conf = config.get(provider
|
|
964
|
+
old_conf = config.get(provider)
|
|
965
965
|
|
|
966
966
|
if old_conf:
|
|
967
967
|
# update non-objects values
|
eodag/plugins/apis/ecmwf.py
CHANGED
|
@@ -123,16 +123,14 @@ class EcmwfApi(Api, ECMWFSearch):
|
|
|
123
123
|
# start date
|
|
124
124
|
if "startTimeFromAscendingNode" not in kwargs:
|
|
125
125
|
kwargs["startTimeFromAscendingNode"] = (
|
|
126
|
-
getattr(self.config, "product_type_config", {}).get(
|
|
127
|
-
"missionStartDate", None
|
|
128
|
-
)
|
|
126
|
+
getattr(self.config, "product_type_config", {}).get("missionStartDate")
|
|
129
127
|
or DEFAULT_MISSION_START_DATE
|
|
130
128
|
)
|
|
131
129
|
# end date
|
|
132
130
|
if "completionTimeFromAscendingNode" not in kwargs:
|
|
133
131
|
kwargs["completionTimeFromAscendingNode"] = getattr(
|
|
134
132
|
self.config, "product_type_config", {}
|
|
135
|
-
).get("missionEndDate"
|
|
133
|
+
).get("missionEndDate") or datetime.now(timezone.utc).isoformat(
|
|
136
134
|
timespec="seconds"
|
|
137
135
|
)
|
|
138
136
|
|
|
@@ -149,8 +147,8 @@ class EcmwfApi(Api, ECMWFSearch):
|
|
|
149
147
|
:raises: :class:`~eodag.utils.exceptions.AuthenticationError`
|
|
150
148
|
"""
|
|
151
149
|
# Get credentials from eodag or using ecmwf conf
|
|
152
|
-
email = getattr(self.config, "credentials", {}).get("username"
|
|
153
|
-
key = getattr(self.config, "credentials", {}).get("password"
|
|
150
|
+
email = getattr(self.config, "credentials", {}).get("username")
|
|
151
|
+
key = getattr(self.config, "credentials", {}).get("password")
|
|
154
152
|
url = getattr(self.config, "auth_endpoint", None)
|
|
155
153
|
if not all([email, key, url]):
|
|
156
154
|
key, url, email = get_apikey_values()
|
|
@@ -295,5 +293,5 @@ class EcmwfApi(Api, ECMWFSearch):
|
|
|
295
293
|
arguments)
|
|
296
294
|
:returns: fetched queryable parameters dict
|
|
297
295
|
"""
|
|
298
|
-
product_type = kwargs.get("productType"
|
|
296
|
+
product_type = kwargs.get("productType")
|
|
299
297
|
return self.queryables_from_metadata_mapping(product_type)
|
eodag/plugins/apis/usgs.py
CHANGED
|
@@ -241,7 +241,7 @@ class UsgsApi(Api):
|
|
|
241
241
|
download_options = api.download_options(
|
|
242
242
|
usgs_dataset, list(results_by_entity_id.keys())
|
|
243
243
|
)
|
|
244
|
-
if download_options.get("data"
|
|
244
|
+
if download_options.get("data") is not None:
|
|
245
245
|
for download_option in download_options["data"]:
|
|
246
246
|
# update results with available downloadSystem
|
|
247
247
|
if (
|
|
@@ -46,7 +46,11 @@ class GenericAuth(Authentication):
|
|
|
46
46
|
"""Authenticate"""
|
|
47
47
|
self.validate_config_credentials()
|
|
48
48
|
method = getattr(self.config, "method", "basic")
|
|
49
|
-
|
|
49
|
+
if not all(x in self.config.credentials for x in ["username", "password"]):
|
|
50
|
+
raise MisconfiguredError(
|
|
51
|
+
f"Missing credentials for provider {self.provider}",
|
|
52
|
+
"You must provide 'username' and 'password' in the configuration.",
|
|
53
|
+
)
|
|
50
54
|
if method == "digest":
|
|
51
55
|
return HTTPDigestAuth(
|
|
52
56
|
self.config.credentials["username"],
|
|
@@ -31,7 +31,12 @@ from requests.auth import AuthBase
|
|
|
31
31
|
|
|
32
32
|
from eodag.plugins.authentication import Authentication
|
|
33
33
|
from eodag.utils import HTTP_REQ_TIMEOUT, USER_AGENT, parse_qs, repeatfunc, urlparse
|
|
34
|
-
from eodag.utils.exceptions import
|
|
34
|
+
from eodag.utils.exceptions import (
|
|
35
|
+
AuthenticationError,
|
|
36
|
+
MisconfiguredError,
|
|
37
|
+
RequestError,
|
|
38
|
+
TimeOutError,
|
|
39
|
+
)
|
|
35
40
|
|
|
36
41
|
if TYPE_CHECKING:
|
|
37
42
|
from requests import PreparedRequest, Response
|
|
@@ -304,12 +309,17 @@ class OIDCAuthorizationCodeFlowAuth(OIDCRefreshTokenBase):
|
|
|
304
309
|
state = self.compute_state()
|
|
305
310
|
authentication_response = self.authenticate_user(state)
|
|
306
311
|
exchange_url = authentication_response.url
|
|
312
|
+
|
|
307
313
|
for err_pattern, err_message in getattr(
|
|
308
314
|
self.config, "exchange_url_error_pattern", {}
|
|
309
315
|
).items():
|
|
310
316
|
if err_pattern in exchange_url:
|
|
311
317
|
raise AuthenticationError(err_message)
|
|
318
|
+
|
|
312
319
|
if not exchange_url.startswith(self.config.redirect_uri):
|
|
320
|
+
if "Invalid username or password" in authentication_response.text:
|
|
321
|
+
raise AuthenticationError("Invalid username or password")
|
|
322
|
+
|
|
313
323
|
raise AuthenticationError(
|
|
314
324
|
f"Could not authenticate user with provider {self.provider}.",
|
|
315
325
|
"Please verify your credentials",
|
|
@@ -369,13 +379,20 @@ class OIDCAuthorizationCodeFlowAuth(OIDCRefreshTokenBase):
|
|
|
369
379
|
"redirect_uri": self.config.redirect_uri,
|
|
370
380
|
}
|
|
371
381
|
ssl_verify = getattr(self.config, "ssl_verify", True)
|
|
372
|
-
|
|
373
|
-
self.
|
|
374
|
-
|
|
375
|
-
|
|
376
|
-
|
|
377
|
-
|
|
378
|
-
|
|
382
|
+
try:
|
|
383
|
+
authorization_response = self.session.get(
|
|
384
|
+
self.authorization_endpoint,
|
|
385
|
+
params=params,
|
|
386
|
+
headers=USER_AGENT,
|
|
387
|
+
timeout=HTTP_REQ_TIMEOUT,
|
|
388
|
+
verify=ssl_verify,
|
|
389
|
+
)
|
|
390
|
+
except requests.exceptions.Timeout as exc:
|
|
391
|
+
raise TimeoutError(exc, "The authentication request timed out.") from exc
|
|
392
|
+
except requests.RequestException as exc:
|
|
393
|
+
raise RequestError.from_error(
|
|
394
|
+
exc, "An error occurred while authenticating the user."
|
|
395
|
+
) from exc
|
|
379
396
|
|
|
380
397
|
login_document = etree.HTML(authorization_response.text)
|
|
381
398
|
login_forms = login_document.xpath(self.config.login_form_xpath)
|
|
@@ -413,13 +430,20 @@ class OIDCAuthorizationCodeFlowAuth(OIDCRefreshTokenBase):
|
|
|
413
430
|
auth_uri = getattr(self.config, "authentication_uri", None)
|
|
414
431
|
if not auth_uri:
|
|
415
432
|
raise MisconfiguredError("authentication_uri is missing")
|
|
416
|
-
|
|
417
|
-
|
|
418
|
-
|
|
419
|
-
|
|
420
|
-
|
|
421
|
-
|
|
422
|
-
|
|
433
|
+
try:
|
|
434
|
+
return self.session.post(
|
|
435
|
+
auth_uri,
|
|
436
|
+
data=login_data,
|
|
437
|
+
headers=USER_AGENT,
|
|
438
|
+
timeout=HTTP_REQ_TIMEOUT,
|
|
439
|
+
verify=ssl_verify,
|
|
440
|
+
)
|
|
441
|
+
except requests.exceptions.Timeout as exc:
|
|
442
|
+
raise TimeoutError(exc, "The authentication request timed out.") from exc
|
|
443
|
+
except requests.RequestException as exc:
|
|
444
|
+
raise RequestError.from_error(
|
|
445
|
+
exc, "An error occurred while authenticating the user."
|
|
446
|
+
) from exc
|
|
423
447
|
|
|
424
448
|
def grant_user_consent(self, authentication_response: Response) -> Response:
|
|
425
449
|
"""Grant user consent"""
|
|
@@ -433,13 +457,20 @@ class OIDCAuthorizationCodeFlowAuth(OIDCRefreshTokenBase):
|
|
|
433
457
|
for key, value in self.config.user_consent_form_data.items()
|
|
434
458
|
}
|
|
435
459
|
ssl_verify = getattr(self.config, "ssl_verify", True)
|
|
436
|
-
|
|
437
|
-
self.
|
|
438
|
-
|
|
439
|
-
|
|
440
|
-
|
|
441
|
-
|
|
442
|
-
|
|
460
|
+
try:
|
|
461
|
+
return self.session.post(
|
|
462
|
+
self.authorization_endpoint,
|
|
463
|
+
data=user_consent_data,
|
|
464
|
+
headers=USER_AGENT,
|
|
465
|
+
timeout=HTTP_REQ_TIMEOUT,
|
|
466
|
+
verify=ssl_verify,
|
|
467
|
+
)
|
|
468
|
+
except requests.exceptions.Timeout as exc:
|
|
469
|
+
raise TimeoutError(exc, "The authentication request timed out.") from exc
|
|
470
|
+
except requests.RequestException as exc:
|
|
471
|
+
raise RequestError.from_error(
|
|
472
|
+
exc, "An error occurred while authenticating the user."
|
|
473
|
+
) from exc
|
|
443
474
|
|
|
444
475
|
def _prepare_token_post_data(self, token_data: dict[str, Any]) -> dict[str, Any]:
|
|
445
476
|
"""Prepare the common data to post to the token URI"""
|
|
@@ -487,14 +518,21 @@ class OIDCAuthorizationCodeFlowAuth(OIDCRefreshTokenBase):
|
|
|
487
518
|
self.config.token_exchange_post_data_method: token_exchange_data
|
|
488
519
|
}
|
|
489
520
|
ssl_verify = getattr(self.config, "ssl_verify", True)
|
|
490
|
-
|
|
491
|
-
self.
|
|
492
|
-
|
|
493
|
-
|
|
494
|
-
|
|
495
|
-
|
|
496
|
-
|
|
497
|
-
|
|
521
|
+
try:
|
|
522
|
+
r = self.session.post(
|
|
523
|
+
self.token_endpoint,
|
|
524
|
+
headers=USER_AGENT,
|
|
525
|
+
timeout=HTTP_REQ_TIMEOUT,
|
|
526
|
+
verify=ssl_verify,
|
|
527
|
+
**post_request_kwargs,
|
|
528
|
+
)
|
|
529
|
+
return r
|
|
530
|
+
except requests.exceptions.Timeout as exc:
|
|
531
|
+
raise TimeoutError(exc, "The authentication request timed out.") from exc
|
|
532
|
+
except requests.RequestException as exc:
|
|
533
|
+
raise RequestError.from_error(
|
|
534
|
+
exc, "An error occurred while authenticating the user."
|
|
535
|
+
) from exc
|
|
498
536
|
|
|
499
537
|
def _constant_or_xpath_extracted(
|
|
500
538
|
self, value: str, form_element: Any
|
|
@@ -116,7 +116,7 @@ class SASAuth(Authentication):
|
|
|
116
116
|
headers = deepcopy(USER_AGENT)
|
|
117
117
|
|
|
118
118
|
# update headers with subscription key if exists
|
|
119
|
-
apikey = getattr(self.config, "credentials", {}).get("apikey"
|
|
119
|
+
apikey = getattr(self.config, "credentials", {}).get("apikey")
|
|
120
120
|
ssl_verify = getattr(self.config, "ssl_verify", True)
|
|
121
121
|
if apikey:
|
|
122
122
|
headers_update = format_dict_items(self.config.headers, apikey=apikey)
|
|
@@ -69,7 +69,7 @@ class FilterDate(Crunch):
|
|
|
69
69
|
return []
|
|
70
70
|
|
|
71
71
|
# filter start date
|
|
72
|
-
filter_start_str = self.config.__dict__.get("start"
|
|
72
|
+
filter_start_str = self.config.__dict__.get("start")
|
|
73
73
|
if filter_start_str:
|
|
74
74
|
filter_start = dateutil.parser.parse(filter_start_str)
|
|
75
75
|
if not filter_start.tzinfo:
|
|
@@ -78,7 +78,7 @@ class FilterDate(Crunch):
|
|
|
78
78
|
filter_start = None
|
|
79
79
|
|
|
80
80
|
# filter end date
|
|
81
|
-
filter_end_str = self.config.__dict__.get("end"
|
|
81
|
+
filter_end_str = self.config.__dict__.get("end")
|
|
82
82
|
if filter_end_str:
|
|
83
83
|
filter_end = dateutil.parser.parse(filter_end_str)
|
|
84
84
|
if not filter_end.tzinfo:
|
|
@@ -93,9 +93,7 @@ class FilterDate(Crunch):
|
|
|
93
93
|
for product in products:
|
|
94
94
|
|
|
95
95
|
# product start date
|
|
96
|
-
product_start_str = product.properties.get(
|
|
97
|
-
"startTimeFromAscendingNode", None
|
|
98
|
-
)
|
|
96
|
+
product_start_str = product.properties.get("startTimeFromAscendingNode")
|
|
99
97
|
if product_start_str:
|
|
100
98
|
product_start = dateutil.parser.parse(product_start_str)
|
|
101
99
|
if not product_start.tzinfo:
|
|
@@ -104,9 +102,7 @@ class FilterDate(Crunch):
|
|
|
104
102
|
product_start = None
|
|
105
103
|
|
|
106
104
|
# product end date
|
|
107
|
-
product_end_str = product.properties.get(
|
|
108
|
-
"completionTimeFromAscendingNode", None
|
|
109
|
-
)
|
|
105
|
+
product_end_str = product.properties.get("completionTimeFromAscendingNode")
|
|
110
106
|
if product_end_str:
|
|
111
107
|
product_end = dateutil.parser.parse(product_end_str)
|
|
112
108
|
if not product_end.tzinfo:
|
|
@@ -64,7 +64,7 @@ class FilterProperty(Crunch):
|
|
|
64
64
|
return products
|
|
65
65
|
|
|
66
66
|
property_key = next(iter(self.config.__dict__))
|
|
67
|
-
property_value = self.config.__dict__.get(property_key
|
|
67
|
+
property_value = self.config.__dict__.get(property_key)
|
|
68
68
|
|
|
69
69
|
logger.debug(
|
|
70
70
|
"Start filtering for products matching operator.%s(product.properties['%s'], %s)",
|
eodag/plugins/download/aws.py
CHANGED
|
@@ -289,7 +289,7 @@ class AwsDownload(Download):
|
|
|
289
289
|
)
|
|
290
290
|
|
|
291
291
|
# do not try to build SAFE if asset filter is used
|
|
292
|
-
asset_filter = kwargs.get("asset"
|
|
292
|
+
asset_filter = kwargs.get("asset")
|
|
293
293
|
if asset_filter:
|
|
294
294
|
build_safe = False
|
|
295
295
|
ignore_assets = False
|
|
@@ -736,7 +736,7 @@ class AwsDownload(Download):
|
|
|
736
736
|
product.product_type, {}
|
|
737
737
|
)
|
|
738
738
|
# do not try to build SAFE if asset filter is used
|
|
739
|
-
asset_filter = kwargs.get("asset"
|
|
739
|
+
asset_filter = kwargs.get("asset")
|
|
740
740
|
if asset_filter:
|
|
741
741
|
build_safe = False
|
|
742
742
|
else:
|
|
@@ -800,7 +800,7 @@ class AwsDownload(Download):
|
|
|
800
800
|
# update headers
|
|
801
801
|
filename = os.path.basename(list(unique_product_chunks)[0].key)
|
|
802
802
|
headers = {"content-disposition": f"attachment; filename={filename}"}
|
|
803
|
-
if assets_values and assets_values[0].get("type"
|
|
803
|
+
if assets_values and assets_values[0].get("type"):
|
|
804
804
|
headers["content-type"] = assets_values[0]["type"]
|
|
805
805
|
|
|
806
806
|
return StreamResponse(
|
eodag/plugins/download/base.py
CHANGED
|
@@ -28,6 +28,7 @@ from datetime import datetime, timedelta
|
|
|
28
28
|
from time import sleep
|
|
29
29
|
from typing import TYPE_CHECKING, Any, Callable, Optional, TypeVar, Union
|
|
30
30
|
|
|
31
|
+
from eodag.api.product.metadata_mapping import ONLINE_STATUS
|
|
31
32
|
from eodag.plugins.base import PluginTopic
|
|
32
33
|
from eodag.utils import (
|
|
33
34
|
DEFAULT_DOWNLOAD_TIMEOUT,
|
|
@@ -600,15 +601,21 @@ class Download(PluginTopic):
|
|
|
600
601
|
if datetime_now >= product.next_try:
|
|
601
602
|
product.next_try += timedelta(minutes=wait)
|
|
602
603
|
try:
|
|
603
|
-
|
|
604
|
-
|
|
604
|
+
download = order_download(*args, **kwargs)
|
|
605
605
|
except NotAvailableError as e:
|
|
606
|
-
if not getattr(self.config, "order_enabled", False):
|
|
607
|
-
raise NotAvailableError(
|
|
608
|
-
f"Product is not available for download and order is not supported for"
|
|
609
|
-
f" {self.provider}, {e}"
|
|
610
|
-
)
|
|
611
606
|
not_available_info = str(e)
|
|
607
|
+
else:
|
|
608
|
+
if (
|
|
609
|
+
product.properties.get("storageStatus", ONLINE_STATUS)
|
|
610
|
+
== ONLINE_STATUS
|
|
611
|
+
) or timeout <= 0:
|
|
612
|
+
return download
|
|
613
|
+
|
|
614
|
+
if not getattr(self.config, "order_enabled", False):
|
|
615
|
+
raise NotAvailableError(
|
|
616
|
+
f"Product is not available for download and order is not supported for"
|
|
617
|
+
f" {self.provider}, {not_available_info}"
|
|
618
|
+
)
|
|
612
619
|
|
|
613
620
|
if datetime_now >= product.next_try and datetime_now < stop_time:
|
|
614
621
|
wait_seconds: Union[float, int] = (
|
eodag/plugins/download/http.py
CHANGED
|
@@ -77,6 +77,7 @@ from eodag.utils.exceptions import (
|
|
|
77
77
|
)
|
|
78
78
|
|
|
79
79
|
if TYPE_CHECKING:
|
|
80
|
+
from jsonpath_ng import JSONPath
|
|
80
81
|
from requests import Response
|
|
81
82
|
|
|
82
83
|
from eodag.api.product import Asset, EOProduct # type: ignore
|
|
@@ -538,7 +539,9 @@ class HTTPDownload(Download):
|
|
|
538
539
|
else {}
|
|
539
540
|
)
|
|
540
541
|
if result_entry:
|
|
541
|
-
entry_jsonpath = string_to_jsonpath(
|
|
542
|
+
entry_jsonpath: JSONPath = string_to_jsonpath(
|
|
543
|
+
result_entry, force=True
|
|
544
|
+
)
|
|
542
545
|
json_response = entry_jsonpath.find(json_response)
|
|
543
546
|
raise NotImplementedError(
|
|
544
547
|
'result_entry in config.on_success is not yet supported for result_type "json"'
|
|
@@ -611,7 +614,7 @@ class HTTPDownload(Download):
|
|
|
611
614
|
# download assets if exist instead of remote_location
|
|
612
615
|
if len(product.assets) > 0 and (
|
|
613
616
|
not getattr(self.config, "ignore_assets", False)
|
|
614
|
-
or kwargs.get("asset"
|
|
617
|
+
or kwargs.get("asset") is not None
|
|
615
618
|
):
|
|
616
619
|
try:
|
|
617
620
|
fs_path = self._download_assets(
|
|
@@ -622,11 +625,11 @@ class HTTPDownload(Download):
|
|
|
622
625
|
progress_callback,
|
|
623
626
|
**kwargs,
|
|
624
627
|
)
|
|
625
|
-
if kwargs.get("asset"
|
|
628
|
+
if kwargs.get("asset") is None:
|
|
626
629
|
product.location = path_to_uri(fs_path)
|
|
627
630
|
return fs_path
|
|
628
631
|
except NotAvailableError as e:
|
|
629
|
-
if kwargs.get("asset"
|
|
632
|
+
if kwargs.get("asset") is not None:
|
|
630
633
|
raise NotAvailableError(e).with_traceback(e.__traceback__)
|
|
631
634
|
else:
|
|
632
635
|
pass
|
|
@@ -661,6 +664,9 @@ class HTTPDownload(Download):
|
|
|
661
664
|
|
|
662
665
|
if is_empty:
|
|
663
666
|
raise DownloadError(f"product {product.properties['id']} is empty")
|
|
667
|
+
else:
|
|
668
|
+
# make sure storage status is online
|
|
669
|
+
product.properties["storageStatus"] = ONLINE_STATUS
|
|
664
670
|
|
|
665
671
|
return path
|
|
666
672
|
else:
|
|
@@ -718,7 +724,7 @@ class HTTPDownload(Download):
|
|
|
718
724
|
|
|
719
725
|
def _check_product_filename(self, product: EOProduct) -> str:
|
|
720
726
|
filename = None
|
|
721
|
-
asset_content_disposition = self.stream.headers.get("content-disposition"
|
|
727
|
+
asset_content_disposition = self.stream.headers.get("content-disposition")
|
|
722
728
|
if asset_content_disposition:
|
|
723
729
|
filename = cast(
|
|
724
730
|
Optional[str],
|
|
@@ -769,7 +775,7 @@ class HTTPDownload(Download):
|
|
|
769
775
|
or kwargs.get("asset") is not None
|
|
770
776
|
):
|
|
771
777
|
try:
|
|
772
|
-
assets_values = product.assets.get_values(kwargs.get("asset"
|
|
778
|
+
assets_values = product.assets.get_values(kwargs.get("asset"))
|
|
773
779
|
chunks_tuples = self._stream_download_assets(
|
|
774
780
|
product,
|
|
775
781
|
auth,
|
|
@@ -786,7 +792,7 @@ class HTTPDownload(Download):
|
|
|
786
792
|
assets_values[0].headers[
|
|
787
793
|
"content-disposition"
|
|
788
794
|
] = f"attachment; filename={assets_values[0].filename}"
|
|
789
|
-
if assets_values[0].get("type"
|
|
795
|
+
if assets_values[0].get("type"):
|
|
790
796
|
assets_values[0].headers["content-type"] = assets_values[0][
|
|
791
797
|
"type"
|
|
792
798
|
]
|
|
@@ -814,7 +820,7 @@ class HTTPDownload(Download):
|
|
|
814
820
|
},
|
|
815
821
|
)
|
|
816
822
|
except NotAvailableError as e:
|
|
817
|
-
if kwargs.get("asset"
|
|
823
|
+
if kwargs.get("asset") is not None:
|
|
818
824
|
raise NotAvailableError(e).with_traceback(e.__traceback__)
|
|
819
825
|
else:
|
|
820
826
|
pass
|
|
@@ -903,7 +909,7 @@ class HTTPDownload(Download):
|
|
|
903
909
|
self._order(product=product, auth=auth)
|
|
904
910
|
|
|
905
911
|
if (
|
|
906
|
-
product.properties.get("orderStatusLink"
|
|
912
|
+
product.properties.get("orderStatusLink")
|
|
907
913
|
and product.properties.get("storageStatus") != ONLINE_STATUS
|
|
908
914
|
):
|
|
909
915
|
self._order_status(product=product, auth=auth)
|
|
@@ -1151,7 +1157,7 @@ class HTTPDownload(Download):
|
|
|
1151
1157
|
if not getattr(asset, "filename", None):
|
|
1152
1158
|
# try getting filename in GET header if was not found in HEAD result
|
|
1153
1159
|
asset_content_disposition = stream.headers.get(
|
|
1154
|
-
"content-disposition"
|
|
1160
|
+
"content-disposition"
|
|
1155
1161
|
)
|
|
1156
1162
|
if asset_content_disposition:
|
|
1157
1163
|
asset.filename = cast(
|
|
@@ -1203,7 +1209,7 @@ class HTTPDownload(Download):
|
|
|
1203
1209
|
if not assets_urls:
|
|
1204
1210
|
raise NotAvailableError("No assets available for %s" % product)
|
|
1205
1211
|
|
|
1206
|
-
assets_values = product.assets.get_values(kwargs.get("asset"
|
|
1212
|
+
assets_values = product.assets.get_values(kwargs.get("asset"))
|
|
1207
1213
|
|
|
1208
1214
|
chunks_tuples = self._stream_download_assets(
|
|
1209
1215
|
product, auth, progress_callback, assets_values=assets_values, **kwargs
|
|
@@ -1289,7 +1295,7 @@ class HTTPDownload(Download):
|
|
|
1289
1295
|
if flatten_top_dirs:
|
|
1290
1296
|
flatten_top_directories(fs_dir_path)
|
|
1291
1297
|
|
|
1292
|
-
if kwargs.get("asset"
|
|
1298
|
+
if kwargs.get("asset") is None:
|
|
1293
1299
|
# save hash/record file
|
|
1294
1300
|
with open(record_filename, "w") as fh:
|
|
1295
1301
|
fh.write(product.remote_location)
|