eodag 3.0.1__py3-none-any.whl → 3.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- eodag/api/core.py +174 -138
- eodag/api/product/_assets.py +44 -15
- eodag/api/product/_product.py +58 -47
- eodag/api/product/drivers/__init__.py +81 -4
- eodag/api/product/drivers/base.py +65 -4
- eodag/api/product/drivers/generic.py +65 -0
- eodag/api/product/drivers/sentinel1.py +97 -0
- eodag/api/product/drivers/sentinel2.py +95 -0
- eodag/api/product/metadata_mapping.py +117 -90
- eodag/api/search_result.py +13 -23
- eodag/cli.py +26 -5
- eodag/config.py +86 -92
- eodag/plugins/apis/base.py +1 -1
- eodag/plugins/apis/ecmwf.py +42 -22
- eodag/plugins/apis/usgs.py +17 -16
- eodag/plugins/authentication/aws_auth.py +16 -13
- eodag/plugins/authentication/base.py +5 -3
- eodag/plugins/authentication/header.py +3 -3
- eodag/plugins/authentication/keycloak.py +4 -4
- eodag/plugins/authentication/oauth.py +7 -3
- eodag/plugins/authentication/openid_connect.py +22 -16
- eodag/plugins/authentication/sas_auth.py +4 -4
- eodag/plugins/authentication/token.py +41 -10
- eodag/plugins/authentication/token_exchange.py +1 -1
- eodag/plugins/base.py +4 -4
- eodag/plugins/crunch/base.py +4 -4
- eodag/plugins/crunch/filter_date.py +4 -4
- eodag/plugins/crunch/filter_latest_intersect.py +6 -6
- eodag/plugins/crunch/filter_latest_tpl_name.py +7 -7
- eodag/plugins/crunch/filter_overlap.py +4 -4
- eodag/plugins/crunch/filter_property.py +6 -7
- eodag/plugins/download/aws.py +146 -87
- eodag/plugins/download/base.py +38 -56
- eodag/plugins/download/creodias_s3.py +29 -0
- eodag/plugins/download/http.py +173 -183
- eodag/plugins/download/s3rest.py +10 -11
- eodag/plugins/manager.py +10 -20
- eodag/plugins/search/__init__.py +6 -5
- eodag/plugins/search/base.py +90 -46
- eodag/plugins/search/build_search_result.py +1048 -361
- eodag/plugins/search/cop_marine.py +22 -12
- eodag/plugins/search/creodias_s3.py +9 -73
- eodag/plugins/search/csw.py +11 -11
- eodag/plugins/search/data_request_search.py +19 -18
- eodag/plugins/search/qssearch.py +99 -258
- eodag/plugins/search/stac_list_assets.py +85 -0
- eodag/plugins/search/static_stac_search.py +4 -4
- eodag/resources/ext_product_types.json +1 -1
- eodag/resources/product_types.yml +1134 -325
- eodag/resources/providers.yml +906 -2006
- eodag/resources/stac_api.yml +2 -2
- eodag/resources/user_conf_template.yml +10 -9
- eodag/rest/cache.py +2 -2
- eodag/rest/config.py +3 -3
- eodag/rest/core.py +112 -82
- eodag/rest/errors.py +5 -5
- eodag/rest/server.py +33 -14
- eodag/rest/stac.py +41 -38
- eodag/rest/types/collections_search.py +3 -3
- eodag/rest/types/eodag_search.py +29 -23
- eodag/rest/types/queryables.py +42 -31
- eodag/rest/types/stac_search.py +15 -25
- eodag/rest/utils/__init__.py +14 -21
- eodag/rest/utils/cql_evaluate.py +6 -6
- eodag/rest/utils/rfc3339.py +2 -2
- eodag/types/__init__.py +141 -32
- eodag/types/bbox.py +2 -2
- eodag/types/download_args.py +3 -3
- eodag/types/queryables.py +183 -72
- eodag/types/search_args.py +4 -4
- eodag/types/whoosh.py +127 -3
- eodag/utils/__init__.py +153 -51
- eodag/utils/exceptions.py +28 -21
- eodag/utils/import_system.py +2 -2
- eodag/utils/repr.py +65 -6
- eodag/utils/requests.py +13 -13
- eodag/utils/rest.py +2 -2
- eodag/utils/s3.py +231 -0
- eodag/utils/stac_reader.py +10 -10
- {eodag-3.0.1.dist-info → eodag-3.1.0.dist-info}/METADATA +77 -76
- eodag-3.1.0.dist-info/RECORD +113 -0
- {eodag-3.0.1.dist-info → eodag-3.1.0.dist-info}/WHEEL +1 -1
- {eodag-3.0.1.dist-info → eodag-3.1.0.dist-info}/entry_points.txt +4 -2
- eodag/utils/constraints.py +0 -244
- eodag-3.0.1.dist-info/RECORD +0 -109
- {eodag-3.0.1.dist-info → eodag-3.1.0.dist-info}/LICENSE +0 -0
- {eodag-3.0.1.dist-info → eodag-3.1.0.dist-info}/top_level.txt +0 -0
eodag/plugins/download/http.py
CHANGED
|
@@ -27,17 +27,8 @@ from datetime import datetime
|
|
|
27
27
|
from email.message import Message
|
|
28
28
|
from itertools import chain
|
|
29
29
|
from json import JSONDecodeError
|
|
30
|
-
from
|
|
31
|
-
|
|
32
|
-
Any,
|
|
33
|
-
Dict,
|
|
34
|
-
Iterator,
|
|
35
|
-
List,
|
|
36
|
-
Optional,
|
|
37
|
-
TypedDict,
|
|
38
|
-
Union,
|
|
39
|
-
cast,
|
|
40
|
-
)
|
|
30
|
+
from pathlib import Path
|
|
31
|
+
from typing import TYPE_CHECKING, Any, Iterator, Optional, TypedDict, Union, cast
|
|
41
32
|
from urllib.parse import parse_qs, urlparse
|
|
42
33
|
|
|
43
34
|
import geojson
|
|
@@ -71,6 +62,7 @@ from eodag.utils import (
|
|
|
71
62
|
guess_file_type,
|
|
72
63
|
parse_header,
|
|
73
64
|
path_to_uri,
|
|
65
|
+
rename_with_version,
|
|
74
66
|
sanitize,
|
|
75
67
|
string_to_jsonpath,
|
|
76
68
|
uri_to_path,
|
|
@@ -80,8 +72,8 @@ from eodag.utils.exceptions import (
|
|
|
80
72
|
DownloadError,
|
|
81
73
|
MisconfiguredError,
|
|
82
74
|
NotAvailableError,
|
|
83
|
-
RequestError,
|
|
84
75
|
TimeOutError,
|
|
76
|
+
ValidationError,
|
|
85
77
|
)
|
|
86
78
|
|
|
87
79
|
if TYPE_CHECKING:
|
|
@@ -90,6 +82,7 @@ if TYPE_CHECKING:
|
|
|
90
82
|
from eodag.api.product import Asset, EOProduct # type: ignore
|
|
91
83
|
from eodag.api.search_result import SearchResult
|
|
92
84
|
from eodag.config import PluginConfig
|
|
85
|
+
from eodag.types import S3SessionKwargs
|
|
93
86
|
from eodag.types.download_args import DownloadConf
|
|
94
87
|
from eodag.utils import DownloadedCallback, Unpack
|
|
95
88
|
|
|
@@ -110,7 +103,7 @@ class HTTPDownload(Download):
|
|
|
110
103
|
extracted; default: ``True``
|
|
111
104
|
* :attr:`~eodag.config.PluginConfig.auth_error_code` (``int``): which error code is returned in case of an
|
|
112
105
|
authentication error
|
|
113
|
-
* :attr:`~eodag.config.PluginConfig.dl_url_params` (``
|
|
106
|
+
* :attr:`~eodag.config.PluginConfig.dl_url_params` (``dict[str, Any]``): parameters to be
|
|
114
107
|
added to the query params of the request
|
|
115
108
|
* :attr:`~eodag.config.PluginConfig.archive_depth` (``int``): level in extracted path tree where to find data;
|
|
116
109
|
default: ``1``
|
|
@@ -122,8 +115,6 @@ class HTTPDownload(Download):
|
|
|
122
115
|
default: ``5``
|
|
123
116
|
* :attr:`~eodag.config.PluginConfig.ssl_verify` (``bool``): if the ssl certificates should be verified in
|
|
124
117
|
requests; default: ``True``
|
|
125
|
-
* :attr:`~eodag.config.PluginConfig.output_extension` (``str``): which extension should be used for the
|
|
126
|
-
downloaded file
|
|
127
118
|
* :attr:`~eodag.config.PluginConfig.no_auth_download` (``bool``): if the download should be done without
|
|
128
119
|
authentication; default: ``True``
|
|
129
120
|
* :attr:`~eodag.config.PluginConfig.order_enabled` (``bool``): if the product has to be ordered to download it;
|
|
@@ -131,7 +122,7 @@ class HTTPDownload(Download):
|
|
|
131
122
|
the search plugin used for the provider; default: ``False``
|
|
132
123
|
* :attr:`~eodag.config.PluginConfig.order_method` (``str``): HTTP request method for the order request (``GET``
|
|
133
124
|
or ``POST``); default: ``GET``
|
|
134
|
-
* :attr:`~eodag.config.PluginConfig.order_headers` (``[
|
|
125
|
+
* :attr:`~eodag.config.PluginConfig.order_headers` (``[dict[str, str]]``): headers to be added to the order
|
|
135
126
|
request
|
|
136
127
|
* :attr:`~eodag.config.PluginConfig.order_on_response` (:class:`~eodag.config.PluginConfig.OrderOnResponse`):
|
|
137
128
|
a typed dictionary containing the key ``metadata_mapping`` which can be used to add new product properties
|
|
@@ -139,22 +130,21 @@ class HTTPDownload(Download):
|
|
|
139
130
|
* :attr:`~eodag.config.PluginConfig.order_status` (:class:`~eodag.config.PluginConfig.OrderStatus`):
|
|
140
131
|
configuration to handle the order status; contains information which method to use, how the response data is
|
|
141
132
|
interpreted, which status corresponds to success, ordered and error and what should be done on success.
|
|
142
|
-
* :attr:`~eodag.config.PluginConfig.products` (``
|
|
143
|
-
keys are the product types, the values are dictionaries which can contain the
|
|
144
|
-
:attr:`~eodag.config.PluginConfig.
|
|
145
|
-
overwrite the provider config for a specific product type
|
|
133
|
+
* :attr:`~eodag.config.PluginConfig.products` (``dict[str, dict[str, Any]``): product type specific config; the
|
|
134
|
+
keys are the product types, the values are dictionaries which can contain the key
|
|
135
|
+
:attr:`~eodag.config.PluginConfig.extract` to overwrite the provider config for a specific product type
|
|
146
136
|
|
|
147
137
|
"""
|
|
148
138
|
|
|
149
139
|
def __init__(self, provider: str, config: PluginConfig) -> None:
|
|
150
140
|
super(HTTPDownload, self).__init__(provider, config)
|
|
151
141
|
|
|
152
|
-
def
|
|
142
|
+
def _order(
|
|
153
143
|
self,
|
|
154
144
|
product: EOProduct,
|
|
155
145
|
auth: Optional[AuthBase] = None,
|
|
156
146
|
**kwargs: Unpack[DownloadConf],
|
|
157
|
-
) -> Optional[
|
|
147
|
+
) -> Optional[dict[str, Any]]:
|
|
158
148
|
"""Send product order request.
|
|
159
149
|
|
|
160
150
|
It will be executed once before the download retry loop, if the product is OFFLINE
|
|
@@ -186,7 +176,7 @@ class HTTPDownload(Download):
|
|
|
186
176
|
ssl_verify = getattr(self.config, "ssl_verify", True)
|
|
187
177
|
timeout = getattr(self.config, "timeout", HTTP_REQ_TIMEOUT)
|
|
188
178
|
OrderKwargs = TypedDict(
|
|
189
|
-
"OrderKwargs", {"json":
|
|
179
|
+
"OrderKwargs", {"json": dict[str, Union[Any, list[str]]]}, total=False
|
|
190
180
|
)
|
|
191
181
|
order_kwargs: OrderKwargs = {}
|
|
192
182
|
if order_method == "POST":
|
|
@@ -226,9 +216,11 @@ class HTTPDownload(Download):
|
|
|
226
216
|
product.properties["storageStatus"] = STAGING_STATUS
|
|
227
217
|
except RequestException as e:
|
|
228
218
|
self._check_auth_exception(e)
|
|
229
|
-
|
|
230
|
-
|
|
231
|
-
|
|
219
|
+
msg = f'{product.properties["title"]} could not be ordered'
|
|
220
|
+
if e.response is not None and e.response.status_code == 400:
|
|
221
|
+
raise ValidationError.from_error(e, msg) from e
|
|
222
|
+
else:
|
|
223
|
+
raise DownloadError.from_error(e, msg) from e
|
|
232
224
|
|
|
233
225
|
return self.order_response_process(response, product)
|
|
234
226
|
except requests.exceptions.Timeout as exc:
|
|
@@ -236,7 +228,7 @@ class HTTPDownload(Download):
|
|
|
236
228
|
|
|
237
229
|
def order_response_process(
|
|
238
230
|
self, response: Response, product: EOProduct
|
|
239
|
-
) -> Optional[
|
|
231
|
+
) -> Optional[dict[str, Any]]:
|
|
240
232
|
"""Process order response
|
|
241
233
|
|
|
242
234
|
:param response: The order response
|
|
@@ -271,7 +263,7 @@ class HTTPDownload(Download):
|
|
|
271
263
|
|
|
272
264
|
return json_response
|
|
273
265
|
|
|
274
|
-
def
|
|
266
|
+
def _order_status(
|
|
275
267
|
self,
|
|
276
268
|
product: EOProduct,
|
|
277
269
|
auth: Optional[AuthBase] = None,
|
|
@@ -300,7 +292,7 @@ class HTTPDownload(Download):
|
|
|
300
292
|
def _request(
|
|
301
293
|
url: str,
|
|
302
294
|
method: str = "GET",
|
|
303
|
-
headers: Optional[
|
|
295
|
+
headers: Optional[dict[str, Any]] = None,
|
|
304
296
|
json: Optional[Any] = None,
|
|
305
297
|
timeout: int = HTTP_REQ_TIMEOUT,
|
|
306
298
|
) -> Response:
|
|
@@ -336,7 +328,7 @@ class HTTPDownload(Download):
|
|
|
336
328
|
except requests.exceptions.Timeout as exc:
|
|
337
329
|
raise TimeOutError(exc, timeout=timeout) from exc
|
|
338
330
|
|
|
339
|
-
status_request:
|
|
331
|
+
status_request: dict[str, Any] = status_config.get("request", {})
|
|
340
332
|
status_request_method = str(status_request.get("method", "GET")).upper()
|
|
341
333
|
|
|
342
334
|
if status_request_method == "POST":
|
|
@@ -353,8 +345,8 @@ class HTTPDownload(Download):
|
|
|
353
345
|
|
|
354
346
|
# check header for success before full status request
|
|
355
347
|
skip_parsing_status_response = False
|
|
356
|
-
status_dict:
|
|
357
|
-
config_on_success:
|
|
348
|
+
status_dict: dict[str, Any] = {}
|
|
349
|
+
config_on_success: dict[str, Any] = status_config.get("on_success", {})
|
|
358
350
|
on_success_mm = config_on_success.get("metadata_mapping", {})
|
|
359
351
|
|
|
360
352
|
status_response_content_needed = (
|
|
@@ -398,13 +390,11 @@ class HTTPDownload(Download):
|
|
|
398
390
|
# success and no need to get status response content
|
|
399
391
|
skip_parsing_status_response = True
|
|
400
392
|
except RequestException as e:
|
|
401
|
-
|
|
402
|
-
|
|
403
|
-
|
|
404
|
-
|
|
405
|
-
|
|
406
|
-
)
|
|
407
|
-
) from e
|
|
393
|
+
msg = f'{product.properties["title"]} order status could not be checked'
|
|
394
|
+
if e.response is not None and e.response.status_code == 400:
|
|
395
|
+
raise ValidationError.from_error(e, msg) from e
|
|
396
|
+
else:
|
|
397
|
+
raise DownloadError.from_error(e, msg) from e
|
|
408
398
|
|
|
409
399
|
if not skip_parsing_status_response:
|
|
410
400
|
# status request
|
|
@@ -440,13 +430,13 @@ class HTTPDownload(Download):
|
|
|
440
430
|
product.properties["orderStatus"] = status_dict.get("status")
|
|
441
431
|
|
|
442
432
|
# handle status error
|
|
443
|
-
errors:
|
|
433
|
+
errors: dict[str, Any] = status_config.get("error", {})
|
|
444
434
|
if errors and errors.items() <= status_dict.items():
|
|
445
435
|
raise DownloadError(
|
|
446
436
|
f"Provider {product.provider} returned: {status_dict.get('error_message', status_message)}"
|
|
447
437
|
)
|
|
448
438
|
|
|
449
|
-
success_status:
|
|
439
|
+
success_status: dict[str, Any] = status_config.get("success", {}).get("status")
|
|
450
440
|
# if not success
|
|
451
441
|
if (success_status and success_status != status_dict.get("status")) or (
|
|
452
442
|
success_code and success_code != response.status_code
|
|
@@ -564,10 +554,10 @@ class HTTPDownload(Download):
|
|
|
564
554
|
def download(
|
|
565
555
|
self,
|
|
566
556
|
product: EOProduct,
|
|
567
|
-
auth: Optional[Union[AuthBase,
|
|
557
|
+
auth: Optional[Union[AuthBase, S3SessionKwargs]] = None,
|
|
568
558
|
progress_callback: Optional[ProgressCallback] = None,
|
|
569
|
-
wait:
|
|
570
|
-
timeout:
|
|
559
|
+
wait: float = DEFAULT_DOWNLOAD_WAIT,
|
|
560
|
+
timeout: float = DEFAULT_DOWNLOAD_TIMEOUT,
|
|
571
561
|
**kwargs: Unpack[DownloadConf],
|
|
572
562
|
) -> Optional[str]:
|
|
573
563
|
"""Download a product using HTTP protocol.
|
|
@@ -585,13 +575,6 @@ class HTTPDownload(Download):
|
|
|
585
575
|
)
|
|
586
576
|
progress_callback = ProgressCallback(disable=True)
|
|
587
577
|
|
|
588
|
-
output_extension = getattr(self.config, "products", {}).get(
|
|
589
|
-
product.product_type, {}
|
|
590
|
-
).get("output_extension", None) or getattr(
|
|
591
|
-
self.config, "output_extension", ".zip"
|
|
592
|
-
)
|
|
593
|
-
kwargs["output_extension"] = kwargs.get("output_extension", output_extension)
|
|
594
|
-
|
|
595
578
|
fs_path, record_filename = self._prepare_download(
|
|
596
579
|
product,
|
|
597
580
|
progress_callback=progress_callback,
|
|
@@ -610,7 +593,7 @@ class HTTPDownload(Download):
|
|
|
610
593
|
try:
|
|
611
594
|
fs_path = self._download_assets(
|
|
612
595
|
product,
|
|
613
|
-
fs_path
|
|
596
|
+
fs_path,
|
|
614
597
|
record_filename,
|
|
615
598
|
auth,
|
|
616
599
|
progress_callback,
|
|
@@ -627,82 +610,64 @@ class HTTPDownload(Download):
|
|
|
627
610
|
|
|
628
611
|
url = product.remote_location
|
|
629
612
|
|
|
630
|
-
@self.
|
|
613
|
+
@self._order_download_retry(product, wait, timeout)
|
|
631
614
|
def download_request(
|
|
632
615
|
product: EOProduct,
|
|
633
616
|
auth: AuthBase,
|
|
634
617
|
progress_callback: ProgressCallback,
|
|
635
|
-
wait:
|
|
636
|
-
timeout:
|
|
618
|
+
wait: float,
|
|
619
|
+
timeout: float,
|
|
637
620
|
**kwargs: Unpack[DownloadConf],
|
|
638
|
-
) ->
|
|
639
|
-
chunks = self._stream_download(product, auth, progress_callback, **kwargs)
|
|
621
|
+
) -> os.PathLike:
|
|
640
622
|
is_empty = True
|
|
623
|
+
chunk_iterator = self._stream_download(
|
|
624
|
+
product, auth, progress_callback, **kwargs
|
|
625
|
+
)
|
|
626
|
+
if fs_path is not None:
|
|
627
|
+
ext = Path(product.filename).suffix
|
|
628
|
+
path = Path(fs_path).with_suffix(ext)
|
|
641
629
|
|
|
642
|
-
|
|
643
|
-
|
|
644
|
-
|
|
645
|
-
|
|
630
|
+
with open(path, "wb") as fhandle:
|
|
631
|
+
for chunk in chunk_iterator:
|
|
632
|
+
is_empty = False
|
|
633
|
+
progress_callback(len(chunk))
|
|
634
|
+
fhandle.write(chunk)
|
|
635
|
+
self.stream.close() # Closing response stream
|
|
646
636
|
|
|
647
|
-
|
|
648
|
-
|
|
637
|
+
if is_empty:
|
|
638
|
+
raise DownloadError(f"product {product.properties['id']} is empty")
|
|
649
639
|
|
|
650
|
-
|
|
640
|
+
return path
|
|
641
|
+
else:
|
|
642
|
+
raise DownloadError(
|
|
643
|
+
f"download of product {product.properties['id']} failed"
|
|
644
|
+
)
|
|
645
|
+
|
|
646
|
+
path = download_request(
|
|
647
|
+
product, auth, progress_callback, wait, timeout, **kwargs
|
|
648
|
+
)
|
|
651
649
|
|
|
652
650
|
with open(record_filename, "w") as fh:
|
|
653
651
|
fh.write(url)
|
|
654
652
|
logger.debug("Download recorded in %s", record_filename)
|
|
655
653
|
|
|
656
|
-
|
|
657
|
-
|
|
658
|
-
logger.warning(
|
|
659
|
-
"Downloaded product is not a Zip File. Please check its file type before using it"
|
|
660
|
-
)
|
|
661
|
-
new_fs_path = os.path.join(
|
|
662
|
-
os.path.dirname(fs_path),
|
|
663
|
-
sanitize(product.properties["title"]),
|
|
664
|
-
)
|
|
665
|
-
if os.path.isfile(fs_path) and not tarfile.is_tarfile(fs_path):
|
|
666
|
-
if not os.path.isdir(new_fs_path):
|
|
667
|
-
os.makedirs(new_fs_path)
|
|
668
|
-
shutil.move(fs_path, new_fs_path)
|
|
669
|
-
file_path = os.path.join(new_fs_path, os.path.basename(fs_path))
|
|
670
|
-
new_file_path = file_path[: file_path.index(".zip")]
|
|
671
|
-
shutil.move(file_path, new_file_path)
|
|
672
|
-
# in the case where the outputs extension has not been set
|
|
673
|
-
# to ".tar" in the product type nor provider configuration
|
|
674
|
-
elif tarfile.is_tarfile(fs_path):
|
|
675
|
-
if not new_fs_path.endswith(".tar"):
|
|
676
|
-
new_fs_path += ".tar"
|
|
677
|
-
shutil.move(fs_path, new_fs_path)
|
|
678
|
-
kwargs["output_extension"] = ".tar"
|
|
679
|
-
product_path = self._finalize(
|
|
680
|
-
new_fs_path,
|
|
681
|
-
progress_callback=progress_callback,
|
|
682
|
-
**kwargs,
|
|
683
|
-
)
|
|
684
|
-
product.location = path_to_uri(product_path)
|
|
685
|
-
return product_path
|
|
686
|
-
else:
|
|
687
|
-
# not a file (dir with zip extension)
|
|
688
|
-
shutil.move(fs_path, new_fs_path)
|
|
689
|
-
product.location = path_to_uri(new_fs_path)
|
|
690
|
-
return new_fs_path
|
|
691
|
-
|
|
692
|
-
if os.path.isfile(fs_path) and not (
|
|
693
|
-
zipfile.is_zipfile(fs_path) or tarfile.is_tarfile(fs_path)
|
|
654
|
+
if os.path.isfile(path) and not (
|
|
655
|
+
zipfile.is_zipfile(path) or tarfile.is_tarfile(path)
|
|
694
656
|
):
|
|
695
657
|
new_fs_path = os.path.join(
|
|
696
|
-
os.path.dirname(
|
|
658
|
+
os.path.dirname(path),
|
|
697
659
|
sanitize(product.properties["title"]),
|
|
698
660
|
)
|
|
661
|
+
if os.path.isfile(new_fs_path):
|
|
662
|
+
rename_with_version(new_fs_path)
|
|
699
663
|
if not os.path.isdir(new_fs_path):
|
|
700
664
|
os.makedirs(new_fs_path)
|
|
701
|
-
shutil.move(
|
|
665
|
+
shutil.move(path, new_fs_path)
|
|
702
666
|
product.location = path_to_uri(new_fs_path)
|
|
703
667
|
return new_fs_path
|
|
668
|
+
|
|
704
669
|
product_path = self._finalize(
|
|
705
|
-
|
|
670
|
+
str(path),
|
|
706
671
|
progress_callback=progress_callback,
|
|
707
672
|
**kwargs,
|
|
708
673
|
)
|
|
@@ -743,24 +708,15 @@ class HTTPDownload(Download):
|
|
|
743
708
|
ext = guess_extension(content_type)
|
|
744
709
|
if ext:
|
|
745
710
|
filename += ext
|
|
746
|
-
else:
|
|
747
|
-
output_extension: Optional[str] = (
|
|
748
|
-
getattr(self.config, "products", {})
|
|
749
|
-
.get(product.product_type, {})
|
|
750
|
-
.get("output_extension")
|
|
751
|
-
)
|
|
752
|
-
if output_extension:
|
|
753
|
-
filename += output_extension
|
|
754
|
-
|
|
755
711
|
return filename
|
|
756
712
|
|
|
757
713
|
def _stream_download_dict(
|
|
758
714
|
self,
|
|
759
715
|
product: EOProduct,
|
|
760
|
-
auth: Optional[Union[AuthBase,
|
|
716
|
+
auth: Optional[Union[AuthBase, S3SessionKwargs]] = None,
|
|
761
717
|
progress_callback: Optional[ProgressCallback] = None,
|
|
762
|
-
wait:
|
|
763
|
-
timeout:
|
|
718
|
+
wait: float = DEFAULT_DOWNLOAD_WAIT,
|
|
719
|
+
timeout: float = DEFAULT_DOWNLOAD_TIMEOUT,
|
|
764
720
|
**kwargs: Unpack[DownloadConf],
|
|
765
721
|
) -> StreamResponse:
|
|
766
722
|
r"""
|
|
@@ -816,13 +772,17 @@ class HTTPDownload(Download):
|
|
|
816
772
|
)
|
|
817
773
|
|
|
818
774
|
else:
|
|
775
|
+
# get first chunk to check if it does not contain an error (if it does, that error will be raised)
|
|
776
|
+
first_chunks_tuple = next(chunks_tuples)
|
|
819
777
|
outputs_filename = (
|
|
820
778
|
sanitize(product.properties["title"])
|
|
821
779
|
if "title" in product.properties
|
|
822
780
|
else sanitize(product.properties.get("id", "download"))
|
|
823
781
|
)
|
|
824
782
|
return StreamResponse(
|
|
825
|
-
content=stream_zip(
|
|
783
|
+
content=stream_zip(
|
|
784
|
+
chain(iter([first_chunks_tuple]), chunks_tuples)
|
|
785
|
+
),
|
|
826
786
|
media_type="application/zip",
|
|
827
787
|
headers={
|
|
828
788
|
"content-disposition": f"attachment; filename={outputs_filename}.zip",
|
|
@@ -834,17 +794,20 @@ class HTTPDownload(Download):
|
|
|
834
794
|
else:
|
|
835
795
|
pass
|
|
836
796
|
|
|
837
|
-
|
|
797
|
+
chunk_iterator = self._stream_download(
|
|
798
|
+
product, auth, progress_callback, **kwargs
|
|
799
|
+
)
|
|
800
|
+
|
|
838
801
|
# start reading chunks to set product.headers
|
|
839
802
|
try:
|
|
840
|
-
first_chunk = next(
|
|
803
|
+
first_chunk = next(chunk_iterator)
|
|
841
804
|
except StopIteration:
|
|
842
805
|
# product is empty file
|
|
843
806
|
logger.error("product %s is empty", product.properties["id"])
|
|
844
807
|
raise NotAvailableError(f"product {product.properties['id']} is empty")
|
|
845
808
|
|
|
846
809
|
return StreamResponse(
|
|
847
|
-
content=chain(iter([first_chunk]),
|
|
810
|
+
content=chain(iter([first_chunk]), chunk_iterator),
|
|
848
811
|
headers=product.headers,
|
|
849
812
|
)
|
|
850
813
|
|
|
@@ -902,6 +865,44 @@ class HTTPDownload(Download):
|
|
|
902
865
|
else:
|
|
903
866
|
logger.error("Error while getting resource :\n%s", tb.format_exc())
|
|
904
867
|
|
|
868
|
+
def _order_request(
|
|
869
|
+
self,
|
|
870
|
+
product: EOProduct,
|
|
871
|
+
auth: Optional[AuthBase],
|
|
872
|
+
) -> None:
|
|
873
|
+
if (
|
|
874
|
+
"orderLink" in product.properties
|
|
875
|
+
and product.properties.get("storageStatus") == OFFLINE_STATUS
|
|
876
|
+
and not product.properties.get("orderStatus")
|
|
877
|
+
):
|
|
878
|
+
self._order(product=product, auth=auth)
|
|
879
|
+
|
|
880
|
+
if (
|
|
881
|
+
product.properties.get("orderStatusLink", None)
|
|
882
|
+
and product.properties.get("storageStatus") != ONLINE_STATUS
|
|
883
|
+
):
|
|
884
|
+
self._order_status(product=product, auth=auth)
|
|
885
|
+
|
|
886
|
+
def order(
|
|
887
|
+
self,
|
|
888
|
+
product: EOProduct,
|
|
889
|
+
auth: Optional[Union[AuthBase, S3SessionKwargs]] = None,
|
|
890
|
+
wait: float = DEFAULT_DOWNLOAD_WAIT,
|
|
891
|
+
timeout: float = DEFAULT_DOWNLOAD_TIMEOUT,
|
|
892
|
+
) -> None:
|
|
893
|
+
"""
|
|
894
|
+
Order product and poll to check its status
|
|
895
|
+
|
|
896
|
+
:param product: The EO product to download
|
|
897
|
+
:param auth: (optional) authenticated object
|
|
898
|
+
:param wait: (optional) Wait time in minutes between two order status check
|
|
899
|
+
:param timeout: (optional) Maximum time in minutes before stop checking
|
|
900
|
+
order status
|
|
901
|
+
"""
|
|
902
|
+
self._order_download_retry(product, wait, timeout)(self._order_request)(
|
|
903
|
+
product, auth
|
|
904
|
+
)
|
|
905
|
+
|
|
905
906
|
def _stream_download(
|
|
906
907
|
self,
|
|
907
908
|
product: EOProduct,
|
|
@@ -910,8 +911,9 @@ class HTTPDownload(Download):
|
|
|
910
911
|
**kwargs: Unpack[DownloadConf],
|
|
911
912
|
) -> Iterator[Any]:
|
|
912
913
|
"""
|
|
913
|
-
|
|
914
|
+
Fetches a zip file containing the assets of a given product as a stream
|
|
914
915
|
and returns a generator yielding the chunks of the file
|
|
916
|
+
|
|
915
917
|
:param product: product for which the assets should be downloaded
|
|
916
918
|
:param auth: The configuration of a plugin of type Authentication
|
|
917
919
|
:param progress_callback: A method or a callable object
|
|
@@ -928,18 +930,8 @@ class HTTPDownload(Download):
|
|
|
928
930
|
ssl_verify = getattr(self.config, "ssl_verify", True)
|
|
929
931
|
|
|
930
932
|
ordered_message = ""
|
|
931
|
-
|
|
932
|
-
|
|
933
|
-
and product.properties.get("storageStatus") == OFFLINE_STATUS
|
|
934
|
-
and not product.properties.get("orderStatus")
|
|
935
|
-
):
|
|
936
|
-
self.order_download(product=product, auth=auth)
|
|
937
|
-
|
|
938
|
-
if (
|
|
939
|
-
product.properties.get("orderStatusLink", None)
|
|
940
|
-
and product.properties.get("storageStatus") != ONLINE_STATUS
|
|
941
|
-
):
|
|
942
|
-
self.order_download_status(product=product, auth=auth)
|
|
933
|
+
# retry handled at download level
|
|
934
|
+
self._order_request(product, auth)
|
|
943
935
|
|
|
944
936
|
params = kwargs.pop("dl_url_params", None) or getattr(
|
|
945
937
|
self.config, "dl_url_params", {}
|
|
@@ -957,7 +949,7 @@ class HTTPDownload(Download):
|
|
|
957
949
|
if not query_dict and parts.query:
|
|
958
950
|
query_dict = geojson.loads(parts.query)
|
|
959
951
|
req_url = parts._replace(query="").geturl()
|
|
960
|
-
req_kwargs:
|
|
952
|
+
req_kwargs: dict[str, Any] = {"json": query_dict} if query_dict else {}
|
|
961
953
|
else:
|
|
962
954
|
req_url = url
|
|
963
955
|
req_kwargs = {}
|
|
@@ -969,7 +961,7 @@ class HTTPDownload(Download):
|
|
|
969
961
|
auth = None
|
|
970
962
|
|
|
971
963
|
s = requests.Session()
|
|
972
|
-
|
|
964
|
+
self.stream = s.request(
|
|
973
965
|
req_method,
|
|
974
966
|
req_url,
|
|
975
967
|
stream=True,
|
|
@@ -979,55 +971,53 @@ class HTTPDownload(Download):
|
|
|
979
971
|
timeout=DEFAULT_STREAM_REQUESTS_TIMEOUT,
|
|
980
972
|
verify=ssl_verify,
|
|
981
973
|
**req_kwargs,
|
|
982
|
-
)
|
|
983
|
-
|
|
984
|
-
|
|
985
|
-
|
|
986
|
-
|
|
987
|
-
|
|
988
|
-
|
|
989
|
-
|
|
990
|
-
|
|
991
|
-
|
|
992
|
-
|
|
993
|
-
|
|
994
|
-
|
|
995
|
-
|
|
996
|
-
|
|
997
|
-
|
|
998
|
-
|
|
999
|
-
|
|
1000
|
-
|
|
1001
|
-
|
|
1002
|
-
|
|
1003
|
-
|
|
1004
|
-
|
|
1005
|
-
|
|
1006
|
-
|
|
1007
|
-
|
|
1008
|
-
|
|
1009
|
-
|
|
1010
|
-
|
|
1011
|
-
|
|
1012
|
-
|
|
1013
|
-
|
|
1014
|
-
|
|
1015
|
-
|
|
1016
|
-
|
|
1017
|
-
product.headers["Content-Type"] = guessed_content_type
|
|
974
|
+
)
|
|
975
|
+
try:
|
|
976
|
+
self.stream.raise_for_status()
|
|
977
|
+
except requests.exceptions.Timeout as exc:
|
|
978
|
+
raise TimeOutError(exc, timeout=DEFAULT_STREAM_REQUESTS_TIMEOUT) from exc
|
|
979
|
+
except RequestException as e:
|
|
980
|
+
self._process_exception(e, product, ordered_message)
|
|
981
|
+
raise DownloadError(
|
|
982
|
+
f"download of {product.properties['id']} is empty"
|
|
983
|
+
) from e
|
|
984
|
+
else:
|
|
985
|
+
# check if product was ordered
|
|
986
|
+
|
|
987
|
+
if getattr(
|
|
988
|
+
self.stream, "status_code", None
|
|
989
|
+
) is not None and self.stream.status_code == getattr(
|
|
990
|
+
self.config, "order_status", {}
|
|
991
|
+
).get(
|
|
992
|
+
"ordered", {}
|
|
993
|
+
).get(
|
|
994
|
+
"http_code"
|
|
995
|
+
):
|
|
996
|
+
product.properties["storageStatus"] = "ORDERED"
|
|
997
|
+
self._process_exception(None, product, ordered_message)
|
|
998
|
+
stream_size = self._check_stream_size(product) or None
|
|
999
|
+
|
|
1000
|
+
product.headers = self.stream.headers
|
|
1001
|
+
filename = self._check_product_filename(product)
|
|
1002
|
+
product.headers["content-disposition"] = f"attachment; filename={filename}"
|
|
1003
|
+
content_type = product.headers.get("Content-Type")
|
|
1004
|
+
guessed_content_type = (
|
|
1005
|
+
guess_file_type(filename) if filename and not content_type else None
|
|
1006
|
+
)
|
|
1007
|
+
if guessed_content_type is not None:
|
|
1008
|
+
product.headers["Content-Type"] = guessed_content_type
|
|
1018
1009
|
|
|
1019
|
-
|
|
1020
|
-
|
|
1021
|
-
|
|
1022
|
-
|
|
1023
|
-
yield chunk
|
|
1010
|
+
progress_callback.reset(total=stream_size)
|
|
1011
|
+
|
|
1012
|
+
product.filename = filename
|
|
1013
|
+
return self.stream.iter_content(chunk_size=64 * 1024)
|
|
1024
1014
|
|
|
1025
1015
|
def _stream_download_assets(
|
|
1026
1016
|
self,
|
|
1027
1017
|
product: EOProduct,
|
|
1028
1018
|
auth: Optional[AuthBase] = None,
|
|
1029
1019
|
progress_callback: Optional[ProgressCallback] = None,
|
|
1030
|
-
assets_values:
|
|
1020
|
+
assets_values: list[Asset] = [],
|
|
1031
1021
|
**kwargs: Unpack[DownloadConf],
|
|
1032
1022
|
) -> Iterator[Any]:
|
|
1033
1023
|
if progress_callback is None:
|
|
@@ -1297,9 +1287,9 @@ class HTTPDownload(Download):
|
|
|
1297
1287
|
|
|
1298
1288
|
def _get_asset_sizes(
|
|
1299
1289
|
self,
|
|
1300
|
-
assets_values:
|
|
1290
|
+
assets_values: list[Asset],
|
|
1301
1291
|
auth: Optional[AuthBase],
|
|
1302
|
-
params: Optional[
|
|
1292
|
+
params: Optional[dict[str, str]],
|
|
1303
1293
|
zipped: bool = False,
|
|
1304
1294
|
) -> int:
|
|
1305
1295
|
total_size = 0
|
|
@@ -1372,11 +1362,11 @@ class HTTPDownload(Download):
|
|
|
1372
1362
|
def download_all(
|
|
1373
1363
|
self,
|
|
1374
1364
|
products: SearchResult,
|
|
1375
|
-
auth: Optional[Union[AuthBase,
|
|
1365
|
+
auth: Optional[Union[AuthBase, S3SessionKwargs]] = None,
|
|
1376
1366
|
downloaded_callback: Optional[DownloadedCallback] = None,
|
|
1377
1367
|
progress_callback: Optional[ProgressCallback] = None,
|
|
1378
|
-
wait:
|
|
1379
|
-
timeout:
|
|
1368
|
+
wait: float = DEFAULT_DOWNLOAD_WAIT,
|
|
1369
|
+
timeout: float = DEFAULT_DOWNLOAD_TIMEOUT,
|
|
1380
1370
|
**kwargs: Unpack[DownloadConf],
|
|
1381
1371
|
):
|
|
1382
1372
|
"""
|