eodag 2.12.0__py3-none-any.whl → 3.0.0b1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- eodag/api/core.py +434 -319
- eodag/api/product/__init__.py +5 -1
- eodag/api/product/_assets.py +7 -2
- eodag/api/product/_product.py +46 -68
- eodag/api/product/metadata_mapping.py +181 -66
- eodag/api/search_result.py +21 -1
- eodag/cli.py +20 -6
- eodag/config.py +95 -6
- eodag/plugins/apis/base.py +8 -162
- eodag/plugins/apis/ecmwf.py +36 -24
- eodag/plugins/apis/usgs.py +40 -24
- eodag/plugins/authentication/aws_auth.py +2 -2
- eodag/plugins/authentication/header.py +31 -6
- eodag/plugins/authentication/keycloak.py +13 -84
- eodag/plugins/authentication/oauth.py +3 -3
- eodag/plugins/authentication/openid_connect.py +256 -46
- eodag/plugins/authentication/qsauth.py +3 -0
- eodag/plugins/authentication/sas_auth.py +8 -1
- eodag/plugins/authentication/token.py +92 -46
- eodag/plugins/authentication/token_exchange.py +120 -0
- eodag/plugins/download/aws.py +86 -91
- eodag/plugins/download/base.py +72 -40
- eodag/plugins/download/http.py +607 -264
- eodag/plugins/download/s3rest.py +28 -15
- eodag/plugins/manager.py +73 -57
- eodag/plugins/search/__init__.py +36 -0
- eodag/plugins/search/base.py +225 -18
- eodag/plugins/search/build_search_result.py +389 -32
- eodag/plugins/search/cop_marine.py +378 -0
- eodag/plugins/search/creodias_s3.py +15 -14
- eodag/plugins/search/csw.py +5 -7
- eodag/plugins/search/data_request_search.py +44 -20
- eodag/plugins/search/qssearch.py +508 -203
- eodag/plugins/search/static_stac_search.py +99 -36
- eodag/resources/constraints/climate-dt.json +13 -0
- eodag/resources/constraints/extremes-dt.json +8 -0
- eodag/resources/ext_product_types.json +1 -1
- eodag/resources/product_types.yml +1897 -34
- eodag/resources/providers.yml +3539 -3277
- eodag/resources/stac.yml +48 -54
- eodag/resources/stac_api.yml +71 -25
- eodag/resources/stac_provider.yml +5 -0
- eodag/resources/user_conf_template.yml +51 -3
- eodag/rest/__init__.py +6 -0
- eodag/rest/cache.py +70 -0
- eodag/rest/config.py +68 -0
- eodag/rest/constants.py +27 -0
- eodag/rest/core.py +757 -0
- eodag/rest/server.py +397 -258
- eodag/rest/stac.py +438 -307
- eodag/rest/types/collections_search.py +44 -0
- eodag/rest/types/eodag_search.py +232 -43
- eodag/rest/types/{stac_queryables.py → queryables.py} +81 -43
- eodag/rest/types/stac_search.py +277 -0
- eodag/rest/utils/__init__.py +216 -0
- eodag/rest/utils/cql_evaluate.py +119 -0
- eodag/rest/utils/rfc3339.py +65 -0
- eodag/types/__init__.py +99 -9
- eodag/types/bbox.py +15 -14
- eodag/types/download_args.py +31 -0
- eodag/types/search_args.py +58 -7
- eodag/types/whoosh.py +81 -0
- eodag/utils/__init__.py +72 -9
- eodag/utils/constraints.py +37 -37
- eodag/utils/exceptions.py +23 -17
- eodag/utils/requests.py +138 -0
- eodag/utils/rest.py +104 -0
- eodag/utils/stac_reader.py +100 -16
- {eodag-2.12.0.dist-info → eodag-3.0.0b1.dist-info}/METADATA +64 -44
- eodag-3.0.0b1.dist-info/RECORD +109 -0
- {eodag-2.12.0.dist-info → eodag-3.0.0b1.dist-info}/WHEEL +1 -1
- {eodag-2.12.0.dist-info → eodag-3.0.0b1.dist-info}/entry_points.txt +6 -5
- eodag/plugins/apis/cds.py +0 -540
- eodag/rest/utils.py +0 -1133
- eodag-2.12.0.dist-info/RECORD +0 -94
- {eodag-2.12.0.dist-info → eodag-3.0.0b1.dist-info}/LICENSE +0 -0
- {eodag-2.12.0.dist-info → eodag-3.0.0b1.dist-info}/top_level.txt +0 -0
|
@@ -0,0 +1,120 @@
|
|
|
1
|
+
# -*- coding: utf-8 -*-
|
|
2
|
+
# Copyright 2024, CS GROUP - France, https://www.csgroup.eu/
|
|
3
|
+
#
|
|
4
|
+
# This file is part of EODAG project
|
|
5
|
+
# https://www.github.com/CS-SI/EODAG
|
|
6
|
+
#
|
|
7
|
+
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
8
|
+
# you may not use this file except in compliance with the License.
|
|
9
|
+
# You may obtain a copy of the License at
|
|
10
|
+
#
|
|
11
|
+
# http://www.apache.org/licenses/LICENSE-2.0
|
|
12
|
+
#
|
|
13
|
+
# Unless required by applicable law or agreed to in writing, software
|
|
14
|
+
# distributed under the License is distributed on an "AS IS" BASIS,
|
|
15
|
+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
16
|
+
# See the License for the specific language governing permissions and
|
|
17
|
+
# limitations under the License.
|
|
18
|
+
from __future__ import annotations
|
|
19
|
+
|
|
20
|
+
import logging
|
|
21
|
+
|
|
22
|
+
import requests
|
|
23
|
+
from requests import RequestException
|
|
24
|
+
|
|
25
|
+
from eodag.config import PluginConfig
|
|
26
|
+
from eodag.plugins.authentication import Authentication
|
|
27
|
+
from eodag.plugins.authentication.openid_connect import (
|
|
28
|
+
CodeAuthorizedAuth,
|
|
29
|
+
OIDCAuthorizationCodeFlowAuth,
|
|
30
|
+
)
|
|
31
|
+
from eodag.utils import HTTP_REQ_TIMEOUT, USER_AGENT
|
|
32
|
+
from eodag.utils.exceptions import AuthenticationError, MisconfiguredError, TimeOutError
|
|
33
|
+
|
|
34
|
+
logger = logging.getLogger("eodag.auth.token_exchange")
|
|
35
|
+
|
|
36
|
+
|
|
37
|
+
class OIDCTokenExchangeAuth(Authentication):
|
|
38
|
+
"""Token exchange implementation using
|
|
39
|
+
:class:`~eodag.plugins.authentication.openid_connect.OIDCAuthorizationCodeFlowAuth` token as subject.
|
|
40
|
+
|
|
41
|
+
The configuration keys of this plugin are as follows (they have no defaults)::
|
|
42
|
+
|
|
43
|
+
# (mandatory) The full OIDCAuthorizationCodeFlowAuth plugin configuration used to retrieve subject token
|
|
44
|
+
subject:
|
|
45
|
+
|
|
46
|
+
# (mandatory) Identifies the issuer of the subject_token
|
|
47
|
+
subject_issuer:
|
|
48
|
+
|
|
49
|
+
# (mandatory) The url to query to get the authorized token
|
|
50
|
+
token_uri:
|
|
51
|
+
|
|
52
|
+
# (mandatory) The OIDC provider's client ID of the eodag provider
|
|
53
|
+
client_id:
|
|
54
|
+
|
|
55
|
+
# (mandatory) This parameter specifies the target client you want the new token minted for.
|
|
56
|
+
audience:
|
|
57
|
+
|
|
58
|
+
# (mandatory) The key pointing to the token in the json response to the POST request to the token server
|
|
59
|
+
token_key:
|
|
60
|
+
"""
|
|
61
|
+
|
|
62
|
+
GRANT_TYPE = "urn:ietf:params:oauth:grant-type:token-exchange"
|
|
63
|
+
SUBJECT_TOKEN_TYPE = "urn:ietf:params:oauth:token-type:access_token"
|
|
64
|
+
REQUIRED_KEYS = [
|
|
65
|
+
"subject",
|
|
66
|
+
"subject_issuer",
|
|
67
|
+
"token_uri",
|
|
68
|
+
"client_id",
|
|
69
|
+
"audience",
|
|
70
|
+
"token_key",
|
|
71
|
+
]
|
|
72
|
+
|
|
73
|
+
def __init__(self, provider: str, config: PluginConfig) -> None:
|
|
74
|
+
super(OIDCTokenExchangeAuth, self).__init__(provider, config)
|
|
75
|
+
for required_key in self.REQUIRED_KEYS:
|
|
76
|
+
if getattr(self.config, required_key, None) is None:
|
|
77
|
+
raise MisconfiguredError(
|
|
78
|
+
f"Missing required entry for OIDCTokenExchangeAuth configuration: {required_key}"
|
|
79
|
+
)
|
|
80
|
+
self.subject = OIDCAuthorizationCodeFlowAuth(
|
|
81
|
+
provider,
|
|
82
|
+
PluginConfig.from_mapping(
|
|
83
|
+
{
|
|
84
|
+
"credentials": getattr(self.config, "credentials", {}),
|
|
85
|
+
**self.config.subject,
|
|
86
|
+
}
|
|
87
|
+
),
|
|
88
|
+
)
|
|
89
|
+
|
|
90
|
+
def authenticate(self) -> CodeAuthorizedAuth:
|
|
91
|
+
"""Authenticate"""
|
|
92
|
+
logger.debug("Getting subject auth token")
|
|
93
|
+
subject_auth = self.subject.authenticate()
|
|
94
|
+
auth_data = {
|
|
95
|
+
"grant_type": self.GRANT_TYPE,
|
|
96
|
+
"subject_token": subject_auth.token,
|
|
97
|
+
"subject_issuer": self.config.subject_issuer,
|
|
98
|
+
"subject_token_type": self.SUBJECT_TOKEN_TYPE,
|
|
99
|
+
"client_id": self.config.client_id,
|
|
100
|
+
"audience": self.config.audience,
|
|
101
|
+
}
|
|
102
|
+
logger.debug("Getting target auth token")
|
|
103
|
+
try:
|
|
104
|
+
auth_response = self.subject.session.post(
|
|
105
|
+
self.config.token_uri,
|
|
106
|
+
data=auth_data,
|
|
107
|
+
headers=USER_AGENT,
|
|
108
|
+
timeout=HTTP_REQ_TIMEOUT,
|
|
109
|
+
)
|
|
110
|
+
auth_response.raise_for_status()
|
|
111
|
+
except requests.exceptions.Timeout as exc:
|
|
112
|
+
raise TimeOutError(exc, timeout=HTTP_REQ_TIMEOUT) from exc
|
|
113
|
+
except RequestException as exc:
|
|
114
|
+
raise AuthenticationError("Could no get authentication token") from exc
|
|
115
|
+
finally:
|
|
116
|
+
self.subject.session.close()
|
|
117
|
+
|
|
118
|
+
token = auth_response.json()[self.config.token_key]
|
|
119
|
+
|
|
120
|
+
return CodeAuthorizedAuth(token, where="header")
|
eodag/plugins/download/aws.py
CHANGED
|
@@ -43,6 +43,7 @@ import requests
|
|
|
43
43
|
from botocore.exceptions import ClientError, ProfileNotFound
|
|
44
44
|
from botocore.handlers import disable_signing
|
|
45
45
|
from lxml import etree
|
|
46
|
+
from requests.auth import AuthBase
|
|
46
47
|
from stream_zip import ZIP_AUTO, stream_zip
|
|
47
48
|
|
|
48
49
|
from eodag.api.product.metadata_mapping import (
|
|
@@ -57,6 +58,7 @@ from eodag.utils import (
|
|
|
57
58
|
HTTP_REQ_TIMEOUT,
|
|
58
59
|
USER_AGENT,
|
|
59
60
|
ProgressCallback,
|
|
61
|
+
StreamResponse,
|
|
60
62
|
flatten_top_directories,
|
|
61
63
|
get_bucket_name_and_prefix,
|
|
62
64
|
path_to_uri,
|
|
@@ -66,6 +68,7 @@ from eodag.utils import (
|
|
|
66
68
|
from eodag.utils.exceptions import (
|
|
67
69
|
AuthenticationError,
|
|
68
70
|
DownloadError,
|
|
71
|
+
MisconfiguredError,
|
|
69
72
|
NotAvailableError,
|
|
70
73
|
TimeOutError,
|
|
71
74
|
)
|
|
@@ -76,7 +79,8 @@ if TYPE_CHECKING:
|
|
|
76
79
|
from eodag.api.product import EOProduct
|
|
77
80
|
from eodag.api.search_result import SearchResult
|
|
78
81
|
from eodag.config import PluginConfig
|
|
79
|
-
from eodag.
|
|
82
|
+
from eodag.types.download_args import DownloadConf
|
|
83
|
+
from eodag.utils import DownloadedCallback, Unpack
|
|
80
84
|
|
|
81
85
|
|
|
82
86
|
logger = logging.getLogger("eodag.download.aws")
|
|
@@ -230,23 +234,23 @@ class AwsDownload(Download):
|
|
|
230
234
|
def download(
|
|
231
235
|
self,
|
|
232
236
|
product: EOProduct,
|
|
233
|
-
auth: Optional[
|
|
237
|
+
auth: Optional[Union[AuthBase, Dict[str, str]]] = None,
|
|
234
238
|
progress_callback: Optional[ProgressCallback] = None,
|
|
235
239
|
wait: int = DEFAULT_DOWNLOAD_WAIT,
|
|
236
240
|
timeout: int = DEFAULT_DOWNLOAD_TIMEOUT,
|
|
237
|
-
**kwargs:
|
|
238
|
-
) -> str:
|
|
241
|
+
**kwargs: Unpack[DownloadConf],
|
|
242
|
+
) -> Optional[str]:
|
|
239
243
|
"""Download method for AWS S3 API.
|
|
240
244
|
|
|
241
245
|
The product can be downloaded as it is, or as SAFE-formatted product.
|
|
242
246
|
SAFE-build is configured for a given provider and product type.
|
|
243
247
|
If the product title is configured to be updated during download and
|
|
244
248
|
SAFE-formatted, its destination path will be:
|
|
245
|
-
`{outputs_prefix}/{title}
|
|
249
|
+
`{outputs_prefix}/{title}`
|
|
246
250
|
|
|
247
251
|
:param product: The EO product to download
|
|
248
252
|
:type product: :class:`~eodag.api.product._product.EOProduct`
|
|
249
|
-
:param auth: (optional)
|
|
253
|
+
:param auth: (optional) authenticated object
|
|
250
254
|
:type auth: Union[AuthBase, Dict[str, str]]
|
|
251
255
|
:param progress_callback: (optional) A method or a callable object
|
|
252
256
|
which takes a current size and a maximum
|
|
@@ -262,6 +266,11 @@ class AwsDownload(Download):
|
|
|
262
266
|
:returns: The absolute path to the downloaded product in the local filesystem
|
|
263
267
|
:rtype: str
|
|
264
268
|
"""
|
|
269
|
+
if auth is None:
|
|
270
|
+
auth = {}
|
|
271
|
+
if isinstance(auth, AuthBase):
|
|
272
|
+
raise MisconfiguredError("Please use AwsAuth plugin with AwsDownload")
|
|
273
|
+
|
|
265
274
|
if progress_callback is None:
|
|
266
275
|
logger.info(
|
|
267
276
|
"Progress bar unavailable, please call product.download() instead of plugin.download()"
|
|
@@ -272,7 +281,7 @@ class AwsDownload(Download):
|
|
|
272
281
|
product_local_path, record_filename = self._download_preparation(
|
|
273
282
|
product, progress_callback=progress_callback, **kwargs
|
|
274
283
|
)
|
|
275
|
-
if not record_filename:
|
|
284
|
+
if not record_filename or not product_local_path:
|
|
276
285
|
return product_local_path
|
|
277
286
|
|
|
278
287
|
product_conf = getattr(self.config, "products", {}).get(
|
|
@@ -290,7 +299,7 @@ class AwsDownload(Download):
|
|
|
290
299
|
|
|
291
300
|
# product conf overrides provider conf for "flatten_top_dirs"
|
|
292
301
|
flatten_top_dirs = product_conf.get(
|
|
293
|
-
"flatten_top_dirs", getattr(self.config, "flatten_top_dirs",
|
|
302
|
+
"flatten_top_dirs", getattr(self.config, "flatten_top_dirs", True)
|
|
294
303
|
)
|
|
295
304
|
|
|
296
305
|
# xtra metadata needed for SAFE product
|
|
@@ -328,7 +337,7 @@ class AwsDownload(Download):
|
|
|
328
337
|
product,
|
|
329
338
|
)
|
|
330
339
|
|
|
331
|
-
total_size = sum([p.size for p in unique_product_chunks])
|
|
340
|
+
total_size = sum([p.size for p in unique_product_chunks]) or None
|
|
332
341
|
|
|
333
342
|
# download
|
|
334
343
|
progress_callback.reset(total=total_size)
|
|
@@ -384,8 +393,11 @@ class AwsDownload(Download):
|
|
|
384
393
|
return product_local_path
|
|
385
394
|
|
|
386
395
|
def _download_preparation(
|
|
387
|
-
self,
|
|
388
|
-
|
|
396
|
+
self,
|
|
397
|
+
product: EOProduct,
|
|
398
|
+
progress_callback: ProgressCallback,
|
|
399
|
+
**kwargs: Unpack[DownloadConf],
|
|
400
|
+
) -> Tuple[Optional[str], Optional[str]]:
|
|
389
401
|
"""
|
|
390
402
|
preparation for the download:
|
|
391
403
|
- check if file was already downloaded
|
|
@@ -427,6 +439,9 @@ class AwsDownload(Download):
|
|
|
427
439
|
product_conf = getattr(self.config, "products", {}).get(
|
|
428
440
|
product.product_type, {}
|
|
429
441
|
)
|
|
442
|
+
ssl_verify = getattr(self.config, "ssl_verify", True)
|
|
443
|
+
timeout = getattr(self.config, "timeout", HTTP_REQ_TIMEOUT)
|
|
444
|
+
|
|
430
445
|
if build_safe and "fetch_metadata" in product_conf.keys():
|
|
431
446
|
fetch_format = product_conf["fetch_metadata"]["fetch_format"]
|
|
432
447
|
update_metadata = product_conf["fetch_metadata"]["update_metadata"]
|
|
@@ -436,10 +451,13 @@ class AwsDownload(Download):
|
|
|
436
451
|
logger.info("Fetching extra metadata from %s" % fetch_url)
|
|
437
452
|
try:
|
|
438
453
|
resp = requests.get(
|
|
439
|
-
fetch_url,
|
|
454
|
+
fetch_url,
|
|
455
|
+
headers=USER_AGENT,
|
|
456
|
+
timeout=timeout,
|
|
457
|
+
verify=ssl_verify,
|
|
440
458
|
)
|
|
441
459
|
except requests.exceptions.Timeout as exc:
|
|
442
|
-
raise TimeOutError(exc, timeout=
|
|
460
|
+
raise TimeOutError(exc, timeout=timeout) from exc
|
|
443
461
|
update_metadata = mtd_cfg_as_conversion_and_querypath(update_metadata)
|
|
444
462
|
if fetch_format == "json":
|
|
445
463
|
json_resp = resp.json()
|
|
@@ -454,7 +472,10 @@ class AwsDownload(Download):
|
|
|
454
472
|
)
|
|
455
473
|
|
|
456
474
|
def _get_bucket_names_and_prefixes(
|
|
457
|
-
self,
|
|
475
|
+
self,
|
|
476
|
+
product: EOProduct,
|
|
477
|
+
asset_filter: Optional[str] = None,
|
|
478
|
+
ignore_assets: Optional[bool] = False,
|
|
458
479
|
) -> List[Tuple[str, Optional[str]]]:
|
|
459
480
|
"""
|
|
460
481
|
retrieves the bucket names and path prefixes for the assets
|
|
@@ -483,7 +504,7 @@ class AwsDownload(Download):
|
|
|
483
504
|
rf"No asset key matching re.fullmatch(r'{asset_filter}') was found in {product}"
|
|
484
505
|
)
|
|
485
506
|
else:
|
|
486
|
-
assets_values =
|
|
507
|
+
assets_values = product.assets.values()
|
|
487
508
|
|
|
488
509
|
bucket_names_and_prefixes = []
|
|
489
510
|
for complementary_url in assets_values:
|
|
@@ -501,8 +522,8 @@ class AwsDownload(Download):
|
|
|
501
522
|
def _do_authentication(
|
|
502
523
|
self,
|
|
503
524
|
bucket_names_and_prefixes: List[Tuple[str, Optional[str]]],
|
|
504
|
-
auth: Dict[str, str],
|
|
505
|
-
) -> Tuple[Dict[str, Any], ResourceCollection
|
|
525
|
+
auth: Optional[Union[AuthBase, Dict[str, str]]] = None,
|
|
526
|
+
) -> Tuple[Dict[str, Any], ResourceCollection]:
|
|
506
527
|
"""
|
|
507
528
|
authenticates with s3 and retrieves the available objects
|
|
508
529
|
raises an error when authentication is not possible
|
|
@@ -513,11 +534,19 @@ class AwsDownload(Download):
|
|
|
513
534
|
:return: authenticated objects per bucket, list of available objects
|
|
514
535
|
:rtype: Tuple[Dict[str, Any], ResourceCollection[Any]]
|
|
515
536
|
"""
|
|
537
|
+
if not isinstance(auth, (dict, type(None))):
|
|
538
|
+
raise AuthenticationError(
|
|
539
|
+
f"Incompatible authentication information, expected dict or None, got {type(auth)}"
|
|
540
|
+
)
|
|
541
|
+
if auth is None:
|
|
542
|
+
auth = {}
|
|
516
543
|
authenticated_objects: Dict[str, Any] = {}
|
|
517
544
|
auth_error_messages: Set[str] = set()
|
|
518
545
|
for _, pack in enumerate(bucket_names_and_prefixes):
|
|
519
546
|
try:
|
|
520
547
|
bucket_name, prefix = pack
|
|
548
|
+
if not prefix:
|
|
549
|
+
continue
|
|
521
550
|
if bucket_name not in authenticated_objects:
|
|
522
551
|
# get Prefixes longest common base path
|
|
523
552
|
common_prefix = ""
|
|
@@ -532,7 +561,7 @@ class AwsDownload(Download):
|
|
|
532
561
|
[
|
|
533
562
|
p
|
|
534
563
|
for b, p in bucket_names_and_prefixes
|
|
535
|
-
if b == bucket_name and common_prefix in p
|
|
564
|
+
if p and b == bucket_name and common_prefix in p
|
|
536
565
|
]
|
|
537
566
|
)
|
|
538
567
|
< prefixes_in_bucket
|
|
@@ -566,7 +595,7 @@ class AwsDownload(Download):
|
|
|
566
595
|
self,
|
|
567
596
|
bucket_names_and_prefixes: List[Tuple[str, Optional[str]]],
|
|
568
597
|
authenticated_objects: Dict[str, Any],
|
|
569
|
-
asset_filter: str,
|
|
598
|
+
asset_filter: Optional[str],
|
|
570
599
|
ignore_assets: bool,
|
|
571
600
|
product: EOProduct,
|
|
572
601
|
) -> Set[Any]:
|
|
@@ -626,12 +655,12 @@ class AwsDownload(Download):
|
|
|
626
655
|
def _stream_download_dict(
|
|
627
656
|
self,
|
|
628
657
|
product: EOProduct,
|
|
629
|
-
auth: Optional[
|
|
658
|
+
auth: Optional[Union[AuthBase, Dict[str, str]]] = None,
|
|
630
659
|
progress_callback: Optional[ProgressCallback] = None,
|
|
631
660
|
wait: int = DEFAULT_DOWNLOAD_WAIT,
|
|
632
661
|
timeout: int = DEFAULT_DOWNLOAD_TIMEOUT,
|
|
633
|
-
**kwargs:
|
|
634
|
-
) ->
|
|
662
|
+
**kwargs: Unpack[DownloadConf],
|
|
663
|
+
) -> StreamResponse:
|
|
635
664
|
r"""
|
|
636
665
|
Returns dictionnary of :class:`~fastapi.responses.StreamingResponse` keyword-arguments.
|
|
637
666
|
It contains a generator to streamed download chunks and the response headers.
|
|
@@ -725,11 +754,11 @@ class AwsDownload(Download):
|
|
|
725
754
|
if assets_values[0].get("type", None):
|
|
726
755
|
headers["content-type"] = assets_values[0]["type"]
|
|
727
756
|
|
|
728
|
-
return
|
|
757
|
+
return StreamResponse(
|
|
729
758
|
content=chain(iter([first_chunks_tuple]), chunks_tuples),
|
|
730
759
|
headers=headers,
|
|
731
760
|
)
|
|
732
|
-
return
|
|
761
|
+
return StreamResponse(
|
|
733
762
|
content=stream_zip(chunks_tuples),
|
|
734
763
|
media_type="application/zip",
|
|
735
764
|
headers={
|
|
@@ -779,7 +808,7 @@ class AwsDownload(Download):
|
|
|
779
808
|
product.product_type, {}
|
|
780
809
|
)
|
|
781
810
|
flatten_top_dirs = product_conf.get(
|
|
782
|
-
"flatten_top_dirs", getattr(self.config, "flatten_top_dirs",
|
|
811
|
+
"flatten_top_dirs", getattr(self.config, "flatten_top_dirs", True)
|
|
783
812
|
)
|
|
784
813
|
common_path = ""
|
|
785
814
|
if flatten_top_dirs:
|
|
@@ -1154,8 +1183,7 @@ class AwsDownload(Download):
|
|
|
1154
1183
|
# S2 L2A Tile files -----------------------------------------------
|
|
1155
1184
|
if matched := S2L2A_TILE_IMG_REGEX.match(chunk.key):
|
|
1156
1185
|
found_dict = matched.groupdict()
|
|
1157
|
-
product_path = "
|
|
1158
|
-
product.properties["title"],
|
|
1186
|
+
product_path = "GRANULE/%s/IMG_DATA/R%s/T%s%s%s_%s_%s_%s.jp2" % (
|
|
1159
1187
|
found_dict["num"],
|
|
1160
1188
|
found_dict["res"],
|
|
1161
1189
|
found_dict["tile1"],
|
|
@@ -1167,16 +1195,14 @@ class AwsDownload(Download):
|
|
|
1167
1195
|
)
|
|
1168
1196
|
elif matched := S2L2A_TILE_AUX_DIR_REGEX.match(chunk.key):
|
|
1169
1197
|
found_dict = matched.groupdict()
|
|
1170
|
-
product_path = "
|
|
1171
|
-
product.properties["title"],
|
|
1198
|
+
product_path = "GRANULE/%s/AUX_DATA/%s" % (
|
|
1172
1199
|
found_dict["num"],
|
|
1173
1200
|
found_dict["file"],
|
|
1174
1201
|
)
|
|
1175
1202
|
# S2 L2A QI Masks
|
|
1176
1203
|
elif matched := S2_TILE_QI_MSK_REGEX.match(chunk.key):
|
|
1177
1204
|
found_dict = matched.groupdict()
|
|
1178
|
-
product_path = "
|
|
1179
|
-
product.properties["title"],
|
|
1205
|
+
product_path = "GRANULE/%s/QI_DATA/MSK_%sPRB_%s" % (
|
|
1180
1206
|
found_dict["num"],
|
|
1181
1207
|
found_dict["file_base"],
|
|
1182
1208
|
found_dict["file_suffix"],
|
|
@@ -1184,8 +1210,7 @@ class AwsDownload(Download):
|
|
|
1184
1210
|
# S2 L2A QI PVI
|
|
1185
1211
|
elif matched := S2_TILE_QI_PVI_REGEX.match(chunk.key):
|
|
1186
1212
|
found_dict = matched.groupdict()
|
|
1187
|
-
product_path = "
|
|
1188
|
-
product.properties["title"],
|
|
1213
|
+
product_path = "GRANULE/%s/QI_DATA/%s_%s_PVI.jp2" % (
|
|
1189
1214
|
found_dict["num"],
|
|
1190
1215
|
title_part3,
|
|
1191
1216
|
title_date1,
|
|
@@ -1193,15 +1218,13 @@ class AwsDownload(Download):
|
|
|
1193
1218
|
# S2 Tile files ---------------------------------------------------
|
|
1194
1219
|
elif matched := S2_TILE_PREVIEW_DIR_REGEX.match(chunk.key):
|
|
1195
1220
|
found_dict = matched.groupdict()
|
|
1196
|
-
product_path = "
|
|
1197
|
-
product.properties["title"],
|
|
1221
|
+
product_path = "GRANULE/%s/preview/%s" % (
|
|
1198
1222
|
found_dict["num"],
|
|
1199
1223
|
found_dict["file"],
|
|
1200
1224
|
)
|
|
1201
1225
|
elif matched := S2_TILE_IMG_REGEX.match(chunk.key):
|
|
1202
1226
|
found_dict = matched.groupdict()
|
|
1203
|
-
product_path = "
|
|
1204
|
-
product.properties["title"],
|
|
1227
|
+
product_path = "GRANULE/%s/IMG_DATA/T%s%s%s_%s_%s" % (
|
|
1205
1228
|
found_dict["num"],
|
|
1206
1229
|
found_dict["tile1"],
|
|
1207
1230
|
found_dict["tile2"],
|
|
@@ -1211,97 +1234,74 @@ class AwsDownload(Download):
|
|
|
1211
1234
|
)
|
|
1212
1235
|
elif matched := S2_TILE_THUMBNAIL_REGEX.match(chunk.key):
|
|
1213
1236
|
found_dict = matched.groupdict()
|
|
1214
|
-
product_path = "
|
|
1215
|
-
product.properties["title"],
|
|
1237
|
+
product_path = "GRANULE/%s/%s" % (
|
|
1216
1238
|
found_dict["num"],
|
|
1217
1239
|
found_dict["file"],
|
|
1218
1240
|
)
|
|
1219
1241
|
elif matched := S2_TILE_MTD_REGEX.match(chunk.key):
|
|
1220
1242
|
found_dict = matched.groupdict()
|
|
1221
|
-
product_path = "
|
|
1222
|
-
product.properties["title"],
|
|
1223
|
-
found_dict["num"],
|
|
1224
|
-
)
|
|
1243
|
+
product_path = "GRANULE/%s/MTD_TL.xml" % found_dict["num"]
|
|
1225
1244
|
elif matched := S2_TILE_AUX_DIR_REGEX.match(chunk.key):
|
|
1226
1245
|
found_dict = matched.groupdict()
|
|
1227
|
-
product_path = "
|
|
1228
|
-
product.properties["title"],
|
|
1246
|
+
product_path = "GRANULE/%s/AUX_DATA/AUX_%s" % (
|
|
1229
1247
|
found_dict["num"],
|
|
1230
1248
|
found_dict["file"],
|
|
1231
1249
|
)
|
|
1232
1250
|
elif matched := S2_TILE_QI_DIR_REGEX.match(chunk.key):
|
|
1233
1251
|
found_dict = matched.groupdict()
|
|
1234
|
-
product_path = "
|
|
1235
|
-
product.properties["title"],
|
|
1252
|
+
product_path = "GRANULE/%s/QI_DATA/%s" % (
|
|
1236
1253
|
found_dict["num"],
|
|
1237
1254
|
found_dict["file"],
|
|
1238
1255
|
)
|
|
1239
1256
|
# S2 Tiles generic
|
|
1240
1257
|
elif matched := S2_TILE_REGEX.match(chunk.key):
|
|
1241
1258
|
found_dict = matched.groupdict()
|
|
1242
|
-
product_path = "
|
|
1243
|
-
product.properties["title"],
|
|
1259
|
+
product_path = "GRANULE/%s/%s" % (
|
|
1244
1260
|
found_dict["num"],
|
|
1245
1261
|
found_dict["file"],
|
|
1246
1262
|
)
|
|
1247
1263
|
# S2 Product files
|
|
1248
1264
|
elif matched := S2_PROD_DS_MTD_REGEX.match(chunk.key):
|
|
1249
1265
|
found_dict = matched.groupdict()
|
|
1250
|
-
product_path = "
|
|
1251
|
-
product.properties["title"],
|
|
1252
|
-
ds_dir,
|
|
1253
|
-
)
|
|
1266
|
+
product_path = "DATASTRIP/%s/MTD_DS.xml" % ds_dir
|
|
1254
1267
|
elif matched := S2_PROD_DS_QI_REPORT_REGEX.match(chunk.key):
|
|
1255
1268
|
found_dict = matched.groupdict()
|
|
1256
|
-
product_path = "
|
|
1257
|
-
product.properties["title"],
|
|
1269
|
+
product_path = "DATASTRIP/%s/QI_DATA/%s.xml" % (
|
|
1258
1270
|
ds_dir,
|
|
1259
1271
|
found_dict["filename"],
|
|
1260
1272
|
)
|
|
1261
1273
|
elif matched := S2_PROD_DS_QI_REGEX.match(chunk.key):
|
|
1262
1274
|
found_dict = matched.groupdict()
|
|
1263
|
-
product_path = "
|
|
1264
|
-
product.properties["title"],
|
|
1275
|
+
product_path = "DATASTRIP/%s/QI_DATA/%s" % (
|
|
1265
1276
|
ds_dir,
|
|
1266
1277
|
found_dict["file"],
|
|
1267
1278
|
)
|
|
1268
1279
|
elif matched := S2_PROD_INSPIRE_REGEX.match(chunk.key):
|
|
1269
1280
|
found_dict = matched.groupdict()
|
|
1270
|
-
product_path = "
|
|
1281
|
+
product_path = "INSPIRE.xml"
|
|
1271
1282
|
elif matched := S2_PROD_MTD_REGEX.match(chunk.key):
|
|
1272
1283
|
found_dict = matched.groupdict()
|
|
1273
|
-
product_path = "
|
|
1274
|
-
product.properties["title"],
|
|
1275
|
-
s2_processing_level,
|
|
1276
|
-
)
|
|
1284
|
+
product_path = "MTD_MSI%s.xml" % s2_processing_level
|
|
1277
1285
|
# S2 Product generic
|
|
1278
1286
|
elif matched := S2_PROD_REGEX.match(chunk.key):
|
|
1279
1287
|
found_dict = matched.groupdict()
|
|
1280
|
-
product_path = "%s
|
|
1281
|
-
product.properties["title"],
|
|
1282
|
-
found_dict["file"],
|
|
1283
|
-
)
|
|
1288
|
+
product_path = "%s" % found_dict["file"]
|
|
1284
1289
|
# S1 --------------------------------------------------------------
|
|
1285
1290
|
elif matched := S1_CALIB_REGEX.match(chunk.key):
|
|
1286
1291
|
found_dict = matched.groupdict()
|
|
1287
|
-
product_path = (
|
|
1288
|
-
"
|
|
1289
|
-
|
|
1290
|
-
|
|
1291
|
-
|
|
1292
|
-
|
|
1293
|
-
|
|
1294
|
-
found_dict["file_pol"],
|
|
1295
|
-
|
|
1296
|
-
S1_IMG_NB_PER_POLAR.get(
|
|
1297
|
-
product.properties["polarizationMode"], {}
|
|
1298
|
-
).get(found_dict["file_pol"].upper(), 1),
|
|
1299
|
-
)
|
|
1292
|
+
product_path = "annotation/calibration/%s-%s-%s-grd-%s-%s-%03d.xml" % (
|
|
1293
|
+
found_dict["file_prefix"],
|
|
1294
|
+
product.properties["platformSerialIdentifier"].lower(),
|
|
1295
|
+
found_dict["file_beam"],
|
|
1296
|
+
found_dict["file_pol"],
|
|
1297
|
+
s1_title_suffix,
|
|
1298
|
+
S1_IMG_NB_PER_POLAR.get(product.properties["polarizationMode"], {}).get(
|
|
1299
|
+
found_dict["file_pol"].upper(), 1
|
|
1300
|
+
),
|
|
1300
1301
|
)
|
|
1301
1302
|
elif matched := S1_ANNOT_REGEX.match(chunk.key):
|
|
1302
1303
|
found_dict = matched.groupdict()
|
|
1303
|
-
product_path = "
|
|
1304
|
-
product.properties["title"],
|
|
1304
|
+
product_path = "annotation/%s-%s-grd-%s-%s-%03d.xml" % (
|
|
1305
1305
|
product.properties["platformSerialIdentifier"].lower(),
|
|
1306
1306
|
found_dict["file_beam"],
|
|
1307
1307
|
found_dict["file_pol"],
|
|
@@ -1312,8 +1312,7 @@ class AwsDownload(Download):
|
|
|
1312
1312
|
)
|
|
1313
1313
|
elif matched := S1_MEAS_REGEX.match(chunk.key):
|
|
1314
1314
|
found_dict = matched.groupdict()
|
|
1315
|
-
product_path = "
|
|
1316
|
-
product.properties["title"],
|
|
1315
|
+
product_path = "measurement/%s-%s-grd-%s-%s-%03d.%s" % (
|
|
1317
1316
|
product.properties["platformSerialIdentifier"].lower(),
|
|
1318
1317
|
found_dict["file_beam"],
|
|
1319
1318
|
found_dict["file_pol"],
|
|
@@ -1325,18 +1324,14 @@ class AwsDownload(Download):
|
|
|
1325
1324
|
)
|
|
1326
1325
|
elif matched := S1_REPORT_REGEX.match(chunk.key):
|
|
1327
1326
|
found_dict = matched.groupdict()
|
|
1328
|
-
product_path = "%s.SAFE
|
|
1329
|
-
product.properties["title"],
|
|
1327
|
+
product_path = "%s.SAFE-%s" % (
|
|
1330
1328
|
product.properties["title"],
|
|
1331
1329
|
found_dict["file"],
|
|
1332
1330
|
)
|
|
1333
1331
|
# S1 generic
|
|
1334
1332
|
elif matched := S1_REGEX.match(chunk.key):
|
|
1335
1333
|
found_dict = matched.groupdict()
|
|
1336
|
-
product_path = "%s
|
|
1337
|
-
product.properties["title"],
|
|
1338
|
-
found_dict["file"],
|
|
1339
|
-
)
|
|
1334
|
+
product_path = "%s" % found_dict["file"]
|
|
1340
1335
|
# out of SAFE format
|
|
1341
1336
|
else:
|
|
1342
1337
|
raise NotAvailableError(f"Ignored {chunk.key} out of SAFE matching pattern")
|
|
@@ -1347,12 +1342,12 @@ class AwsDownload(Download):
|
|
|
1347
1342
|
def download_all(
|
|
1348
1343
|
self,
|
|
1349
1344
|
products: SearchResult,
|
|
1350
|
-
auth: Optional[
|
|
1345
|
+
auth: Optional[Union[AuthBase, Dict[str, str]]] = None,
|
|
1351
1346
|
downloaded_callback: Optional[DownloadedCallback] = None,
|
|
1352
1347
|
progress_callback: Optional[ProgressCallback] = None,
|
|
1353
1348
|
wait: int = DEFAULT_DOWNLOAD_WAIT,
|
|
1354
1349
|
timeout: int = DEFAULT_DOWNLOAD_TIMEOUT,
|
|
1355
|
-
**kwargs:
|
|
1350
|
+
**kwargs: Unpack[DownloadConf],
|
|
1356
1351
|
) -> List[str]:
|
|
1357
1352
|
"""
|
|
1358
1353
|
download_all using parent (base plugin) method
|