eodag 3.0.0b1__py3-none-any.whl → 3.0.0b3__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- eodag/__init__.py +6 -8
- eodag/api/core.py +119 -171
- eodag/api/product/__init__.py +10 -4
- eodag/api/product/_assets.py +52 -14
- eodag/api/product/_product.py +59 -30
- eodag/api/product/drivers/__init__.py +7 -2
- eodag/api/product/drivers/base.py +0 -3
- eodag/api/product/metadata_mapping.py +0 -28
- eodag/api/search_result.py +31 -9
- eodag/config.py +45 -41
- eodag/plugins/apis/base.py +3 -3
- eodag/plugins/apis/ecmwf.py +2 -3
- eodag/plugins/apis/usgs.py +43 -14
- eodag/plugins/authentication/aws_auth.py +11 -2
- eodag/plugins/authentication/openid_connect.py +5 -4
- eodag/plugins/authentication/token.py +2 -1
- eodag/plugins/crunch/base.py +3 -1
- eodag/plugins/crunch/filter_date.py +3 -9
- eodag/plugins/crunch/filter_latest_intersect.py +0 -3
- eodag/plugins/crunch/filter_latest_tpl_name.py +1 -4
- eodag/plugins/crunch/filter_overlap.py +4 -8
- eodag/plugins/crunch/filter_property.py +5 -11
- eodag/plugins/download/aws.py +46 -78
- eodag/plugins/download/base.py +27 -68
- eodag/plugins/download/http.py +48 -57
- eodag/plugins/download/s3rest.py +17 -25
- eodag/plugins/manager.py +6 -18
- eodag/plugins/search/__init__.py +9 -9
- eodag/plugins/search/base.py +7 -26
- eodag/plugins/search/build_search_result.py +0 -13
- eodag/plugins/search/cop_marine.py +1 -3
- eodag/plugins/search/creodias_s3.py +0 -3
- eodag/plugins/search/data_request_search.py +10 -5
- eodag/plugins/search/qssearch.py +95 -53
- eodag/plugins/search/static_stac_search.py +6 -3
- eodag/resources/ext_product_types.json +1 -1
- eodag/resources/product_types.yml +24 -0
- eodag/resources/providers.yml +198 -154
- eodag/resources/user_conf_template.yml +27 -27
- eodag/rest/core.py +11 -43
- eodag/rest/server.py +1 -6
- eodag/rest/stac.py +13 -87
- eodag/rest/types/eodag_search.py +4 -7
- eodag/rest/types/queryables.py +4 -12
- eodag/rest/types/stac_search.py +7 -11
- eodag/rest/utils/rfc3339.py +0 -1
- eodag/types/__init__.py +9 -3
- eodag/types/download_args.py +14 -5
- eodag/types/search_args.py +7 -8
- eodag/types/whoosh.py +0 -2
- eodag/utils/__init__.py +20 -79
- eodag/utils/constraints.py +0 -8
- eodag/utils/import_system.py +0 -4
- eodag/utils/logging.py +0 -3
- eodag/utils/notebook.py +4 -4
- eodag/utils/repr.py +113 -0
- eodag/utils/requests.py +12 -20
- eodag/utils/rest.py +0 -4
- eodag/utils/stac_reader.py +2 -14
- {eodag-3.0.0b1.dist-info → eodag-3.0.0b3.dist-info}/METADATA +33 -14
- eodag-3.0.0b3.dist-info/RECORD +110 -0
- {eodag-3.0.0b1.dist-info → eodag-3.0.0b3.dist-info}/WHEEL +1 -1
- eodag-3.0.0b1.dist-info/RECORD +0 -109
- {eodag-3.0.0b1.dist-info → eodag-3.0.0b3.dist-info}/LICENSE +0 -0
- {eodag-3.0.0b1.dist-info → eodag-3.0.0b3.dist-info}/entry_points.txt +0 -0
- {eodag-3.0.0b1.dist-info → eodag-3.0.0b3.dist-info}/top_level.txt +0 -0
eodag/plugins/download/http.py
CHANGED
|
@@ -25,6 +25,7 @@ import zipfile
|
|
|
25
25
|
from datetime import datetime
|
|
26
26
|
from email.message import Message
|
|
27
27
|
from itertools import chain
|
|
28
|
+
from json import JSONDecodeError
|
|
28
29
|
from typing import (
|
|
29
30
|
TYPE_CHECKING,
|
|
30
31
|
Any,
|
|
@@ -32,7 +33,6 @@ from typing import (
|
|
|
32
33
|
Iterator,
|
|
33
34
|
List,
|
|
34
35
|
Optional,
|
|
35
|
-
Tuple,
|
|
36
36
|
TypedDict,
|
|
37
37
|
Union,
|
|
38
38
|
cast,
|
|
@@ -44,6 +44,7 @@ import requests
|
|
|
44
44
|
from lxml import etree
|
|
45
45
|
from requests import RequestException
|
|
46
46
|
from requests.auth import AuthBase
|
|
47
|
+
from requests.structures import CaseInsensitiveDict
|
|
47
48
|
from stream_zip import ZIP_AUTO, stream_zip
|
|
48
49
|
|
|
49
50
|
from eodag.api.product.metadata_mapping import (
|
|
@@ -97,7 +98,6 @@ class HTTPDownload(Download):
|
|
|
97
98
|
"""HTTPDownload plugin. Handles product download over HTTP protocol
|
|
98
99
|
|
|
99
100
|
:param provider: provider name
|
|
100
|
-
:type provider: str
|
|
101
101
|
:param config: Download plugin configuration:
|
|
102
102
|
|
|
103
103
|
* ``config.base_uri`` (str) - (optional) default endpoint url
|
|
@@ -113,15 +113,12 @@ class HTTPDownload(Download):
|
|
|
113
113
|
* ``config.order_on_response`` (dict) - (optional) edit or add new product properties
|
|
114
114
|
* ``config.order_status`` (:class:`~eodag.config.PluginConfig.OrderStatus`) - (optional) Order status handling
|
|
115
115
|
|
|
116
|
-
|
|
117
|
-
:type config: :class:`~eodag.config.PluginConfig`
|
|
118
|
-
|
|
119
116
|
"""
|
|
120
117
|
|
|
121
118
|
def __init__(self, provider: str, config: PluginConfig) -> None:
|
|
122
119
|
super(HTTPDownload, self).__init__(provider, config)
|
|
123
120
|
|
|
124
|
-
def
|
|
121
|
+
def order_download(
|
|
125
122
|
self,
|
|
126
123
|
product: EOProduct,
|
|
127
124
|
auth: Optional[AuthBase] = None,
|
|
@@ -147,13 +144,9 @@ class HTTPDownload(Download):
|
|
|
147
144
|
- **orderLink**: order request URL
|
|
148
145
|
|
|
149
146
|
:param product: The EO product to order
|
|
150
|
-
:type product: :class:`~eodag.api.product._product.EOProduct`
|
|
151
147
|
:param auth: (optional) authenticated object
|
|
152
|
-
:type auth: Optional[AuthBase]
|
|
153
148
|
:param kwargs: download additional kwargs
|
|
154
|
-
:type kwargs: Union[str, bool, dict]
|
|
155
149
|
:returns: the returned json status response
|
|
156
|
-
:rtype: dict
|
|
157
150
|
"""
|
|
158
151
|
product.properties["storageStatus"] = STAGING_STATUS
|
|
159
152
|
|
|
@@ -167,10 +160,15 @@ class HTTPDownload(Download):
|
|
|
167
160
|
if order_method == "POST":
|
|
168
161
|
# separate url & parameters
|
|
169
162
|
parts = urlparse(str(product.properties["orderLink"]))
|
|
170
|
-
query_dict =
|
|
171
|
-
|
|
163
|
+
query_dict = {}
|
|
164
|
+
# `parts.query` may be a JSON with query strings as one of values. If `parse_qs` is executed as first step,
|
|
165
|
+
# the resulting `query_dict` would be erroneous.
|
|
166
|
+
try:
|
|
172
167
|
query_dict = geojson.loads(parts.query)
|
|
173
|
-
|
|
168
|
+
except JSONDecodeError:
|
|
169
|
+
if parts.query:
|
|
170
|
+
query_dict = parse_qs(parts.query)
|
|
171
|
+
order_url = parts._replace(query="").geturl()
|
|
174
172
|
if query_dict:
|
|
175
173
|
order_kwargs["json"] = query_dict
|
|
176
174
|
else:
|
|
@@ -217,11 +215,8 @@ class HTTPDownload(Download):
|
|
|
217
215
|
"""Process order response
|
|
218
216
|
|
|
219
217
|
:param response: The order response
|
|
220
|
-
:type response: :class:`~requests.Response`
|
|
221
218
|
:param product: The orderd EO product
|
|
222
|
-
:type product: :class:`~eodag.api.product._product.EOProduct`
|
|
223
219
|
:returns: the returned json status response
|
|
224
|
-
:rtype: dict
|
|
225
220
|
"""
|
|
226
221
|
on_response_mm = getattr(self.config, "order_on_response", {}).get(
|
|
227
222
|
"metadata_mapping", {}
|
|
@@ -251,7 +246,7 @@ class HTTPDownload(Download):
|
|
|
251
246
|
|
|
252
247
|
return json_response
|
|
253
248
|
|
|
254
|
-
def
|
|
249
|
+
def order_download_status(
|
|
255
250
|
self,
|
|
256
251
|
product: EOProduct,
|
|
257
252
|
auth: Optional[AuthBase] = None,
|
|
@@ -268,11 +263,8 @@ class HTTPDownload(Download):
|
|
|
268
263
|
- **orderStatusLink**: order status request URL
|
|
269
264
|
|
|
270
265
|
:param product: The ordered EO product
|
|
271
|
-
:type product: :class:`~eodag.api.product._product.EOProduct`
|
|
272
266
|
:param auth: (optional) authenticated object
|
|
273
|
-
:type auth: Optional[AuthBase]
|
|
274
267
|
:param kwargs: download additional kwargs
|
|
275
|
-
:type kwargs: Union[str, bool, dict]
|
|
276
268
|
"""
|
|
277
269
|
|
|
278
270
|
status_config = getattr(self.config, "order_status", {})
|
|
@@ -325,7 +317,7 @@ class HTTPDownload(Download):
|
|
|
325
317
|
if status_request_method == "POST":
|
|
326
318
|
# separate url & parameters
|
|
327
319
|
parts = urlparse(str(product.properties["orderStatusLink"]))
|
|
328
|
-
status_url = parts._replace(query=
|
|
320
|
+
status_url = parts._replace(query="").geturl()
|
|
329
321
|
query_dict = parse_qs(parts.query)
|
|
330
322
|
if not query_dict and parts.query:
|
|
331
323
|
query_dict = geojson.loads(parts.query)
|
|
@@ -568,12 +560,12 @@ class HTTPDownload(Download):
|
|
|
568
560
|
)
|
|
569
561
|
progress_callback = ProgressCallback(disable=True)
|
|
570
562
|
|
|
571
|
-
|
|
563
|
+
output_extension = getattr(self.config, "products", {}).get(
|
|
572
564
|
product.product_type, {}
|
|
573
|
-
).get("
|
|
574
|
-
self.config, "
|
|
565
|
+
).get("output_extension", None) or getattr(
|
|
566
|
+
self.config, "output_extension", ".zip"
|
|
575
567
|
)
|
|
576
|
-
kwargs["
|
|
568
|
+
kwargs["output_extension"] = kwargs.get("output_extension", output_extension)
|
|
577
569
|
|
|
578
570
|
fs_path, record_filename = self._prepare_download(
|
|
579
571
|
product,
|
|
@@ -637,7 +629,7 @@ class HTTPDownload(Download):
|
|
|
637
629
|
logger.debug("Download recorded in %s", record_filename)
|
|
638
630
|
|
|
639
631
|
# Check that the downloaded file is really a zip file
|
|
640
|
-
if not zipfile.is_zipfile(fs_path) and
|
|
632
|
+
if not zipfile.is_zipfile(fs_path) and output_extension == ".zip":
|
|
641
633
|
logger.warning(
|
|
642
634
|
"Downloaded product is not a Zip File. Please check its file type before using it"
|
|
643
635
|
)
|
|
@@ -658,7 +650,7 @@ class HTTPDownload(Download):
|
|
|
658
650
|
if not new_fs_path.endswith(".tar"):
|
|
659
651
|
new_fs_path += ".tar"
|
|
660
652
|
shutil.move(fs_path, new_fs_path)
|
|
661
|
-
kwargs["
|
|
653
|
+
kwargs["output_extension"] = ".tar"
|
|
662
654
|
product_path = self._finalize(
|
|
663
655
|
new_fs_path,
|
|
664
656
|
progress_callback=progress_callback,
|
|
@@ -727,13 +719,13 @@ class HTTPDownload(Download):
|
|
|
727
719
|
if ext:
|
|
728
720
|
filename += ext
|
|
729
721
|
else:
|
|
730
|
-
|
|
722
|
+
output_extension: Optional[str] = (
|
|
731
723
|
getattr(self.config, "products", {})
|
|
732
724
|
.get(product.product_type, {})
|
|
733
|
-
.get("
|
|
725
|
+
.get("output_extension")
|
|
734
726
|
)
|
|
735
|
-
if
|
|
736
|
-
filename +=
|
|
727
|
+
if output_extension:
|
|
728
|
+
filename += output_extension
|
|
737
729
|
|
|
738
730
|
return filename
|
|
739
731
|
|
|
@@ -751,23 +743,16 @@ class HTTPDownload(Download):
|
|
|
751
743
|
It contains a generator to streamed download chunks and the response headers.
|
|
752
744
|
|
|
753
745
|
:param product: The EO product to download
|
|
754
|
-
:type product: :class:`~eodag.api.product._product.EOProduct`
|
|
755
746
|
:param auth: (optional) authenticated object
|
|
756
|
-
:type auth: Optional[Union[AuthBase, Dict[str, str]]]
|
|
757
747
|
:param progress_callback: (optional) A progress callback
|
|
758
|
-
:type progress_callback: :class:`~eodag.utils.ProgressCallback`
|
|
759
748
|
:param wait: (optional) If download fails, wait time in minutes between two download tries
|
|
760
|
-
:type wait: int
|
|
761
749
|
:param timeout: (optional) If download fails, maximum time in minutes before stop retrying
|
|
762
750
|
to download
|
|
763
|
-
:
|
|
764
|
-
:param kwargs: `outputs_prefix` (str), `extract` (bool), `delete_archive` (bool)
|
|
751
|
+
:param kwargs: `output_dir` (str), `extract` (bool), `delete_archive` (bool)
|
|
765
752
|
and `dl_url_params` (dict) can be provided as additional kwargs
|
|
766
753
|
and will override any other values defined in a configuration
|
|
767
754
|
file or with environment variables.
|
|
768
|
-
:type kwargs: Union[str, bool, dict]
|
|
769
755
|
:returns: Dictionnary of :class:`~fastapi.responses.StreamingResponse` keyword-arguments
|
|
770
|
-
:rtype: dict
|
|
771
756
|
"""
|
|
772
757
|
if auth is not None and not isinstance(auth, AuthBase):
|
|
773
758
|
raise MisconfiguredError(f"Incompatible auth plugin: {type(auth)}")
|
|
@@ -906,17 +891,13 @@ class HTTPDownload(Download):
|
|
|
906
891
|
fetches a zip file containing the assets of a given product as a stream
|
|
907
892
|
and returns a generator yielding the chunks of the file
|
|
908
893
|
:param product: product for which the assets should be downloaded
|
|
909
|
-
:type product: :class:`~eodag.api.product._product.EOProduct`
|
|
910
894
|
:param auth: The configuration of a plugin of type Authentication
|
|
911
|
-
:type auth: Optional[Union[AuthBase, Dict[str, str]]]
|
|
912
895
|
:param progress_callback: A method or a callable object
|
|
913
896
|
which takes a current size and a maximum
|
|
914
897
|
size as inputs and handle progress bar
|
|
915
898
|
creation and update to give the user a
|
|
916
899
|
feedback on the download progress
|
|
917
|
-
:type progress_callback: :class:`~eodag.utils.ProgressCallback`
|
|
918
900
|
:param kwargs: additional arguments
|
|
919
|
-
:type kwargs: dict
|
|
920
901
|
"""
|
|
921
902
|
if progress_callback is None:
|
|
922
903
|
logger.info("Progress bar unavailable, please call product.download()")
|
|
@@ -928,13 +909,13 @@ class HTTPDownload(Download):
|
|
|
928
909
|
and product.properties.get("storageStatus") == OFFLINE_STATUS
|
|
929
910
|
and not product.properties.get("orderStatus")
|
|
930
911
|
):
|
|
931
|
-
self.
|
|
912
|
+
self.order_download(product=product, auth=auth)
|
|
932
913
|
|
|
933
914
|
if (
|
|
934
915
|
product.properties.get("orderStatusLink", None)
|
|
935
916
|
and product.properties.get("storageStatus") != ONLINE_STATUS
|
|
936
917
|
):
|
|
937
|
-
self.
|
|
918
|
+
self.order_download_status(product=product, auth=auth)
|
|
938
919
|
|
|
939
920
|
params = kwargs.pop("dl_url_params", None) or getattr(
|
|
940
921
|
self.config, "dl_url_params", {}
|
|
@@ -951,7 +932,7 @@ class HTTPDownload(Download):
|
|
|
951
932
|
query_dict = parse_qs(parts.query)
|
|
952
933
|
if not query_dict and parts.query:
|
|
953
934
|
query_dict = geojson.loads(parts.query)
|
|
954
|
-
req_url = parts._replace(query=
|
|
935
|
+
req_url = parts._replace(query="").geturl()
|
|
955
936
|
req_kwargs: Dict[str, Any] = {"json": query_dict} if query_dict else {}
|
|
956
937
|
else:
|
|
957
938
|
req_url = url
|
|
@@ -1004,8 +985,11 @@ class HTTPDownload(Download):
|
|
|
1004
985
|
"content-disposition"
|
|
1005
986
|
] = f"attachment; filename={filename}"
|
|
1006
987
|
content_type = product.headers.get("Content-Type")
|
|
1007
|
-
|
|
1008
|
-
|
|
988
|
+
guessed_content_type = (
|
|
989
|
+
guess_file_type(filename) if filename and not content_type else None
|
|
990
|
+
)
|
|
991
|
+
if guessed_content_type is not None:
|
|
992
|
+
product.headers["Content-Type"] = guessed_content_type
|
|
1009
993
|
|
|
1010
994
|
progress_callback.reset(total=stream_size)
|
|
1011
995
|
for chunk in self.stream.iter_content(chunk_size=64 * 1024):
|
|
@@ -1020,7 +1004,7 @@ class HTTPDownload(Download):
|
|
|
1020
1004
|
progress_callback: Optional[ProgressCallback] = None,
|
|
1021
1005
|
assets_values: List[Asset] = [],
|
|
1022
1006
|
**kwargs: Unpack[DownloadConf],
|
|
1023
|
-
) -> Iterator[
|
|
1007
|
+
) -> Iterator[Any]:
|
|
1024
1008
|
if progress_callback is None:
|
|
1025
1009
|
logger.info("Progress bar unavailable, please call product.download()")
|
|
1026
1010
|
progress_callback = ProgressCallback(disable=True)
|
|
@@ -1074,7 +1058,6 @@ class HTTPDownload(Download):
|
|
|
1074
1058
|
|
|
1075
1059
|
# loop for assets download
|
|
1076
1060
|
for asset in assets_values:
|
|
1077
|
-
|
|
1078
1061
|
if not asset["href"] or asset["href"].startswith("file:"):
|
|
1079
1062
|
logger.info(
|
|
1080
1063
|
f"Local asset detected. Download skipped for {asset['href']}"
|
|
@@ -1194,7 +1177,9 @@ class HTTPDownload(Download):
|
|
|
1194
1177
|
# start reading chunks to set asset.rel_path
|
|
1195
1178
|
first_chunks_tuple = next(chunks_tuples)
|
|
1196
1179
|
chunks = chain(iter([first_chunks_tuple]), chunks_tuples)
|
|
1197
|
-
chunks_tuples =
|
|
1180
|
+
chunks_tuples = iter(
|
|
1181
|
+
[(assets_values[0].rel_path, None, None, None, chunks)]
|
|
1182
|
+
)
|
|
1198
1183
|
|
|
1199
1184
|
for chunk_tuple in chunks_tuples:
|
|
1200
1185
|
asset_path = chunk_tuple[0]
|
|
@@ -1291,12 +1276,18 @@ class HTTPDownload(Download):
|
|
|
1291
1276
|
for asset in assets_values:
|
|
1292
1277
|
if asset["href"] and not asset["href"].startswith("file:"):
|
|
1293
1278
|
# HEAD request for size & filename
|
|
1294
|
-
|
|
1295
|
-
|
|
1296
|
-
|
|
1297
|
-
|
|
1298
|
-
|
|
1299
|
-
|
|
1279
|
+
try:
|
|
1280
|
+
asset_headers = requests.head(
|
|
1281
|
+
asset["href"],
|
|
1282
|
+
auth=auth,
|
|
1283
|
+
params=params,
|
|
1284
|
+
headers=USER_AGENT,
|
|
1285
|
+
timeout=timeout,
|
|
1286
|
+
verify=ssl_verify,
|
|
1287
|
+
).headers
|
|
1288
|
+
except RequestException as e:
|
|
1289
|
+
logger.debug(f"HEAD request failed: {str(e)}")
|
|
1290
|
+
asset_headers = CaseInsensitiveDict()
|
|
1300
1291
|
|
|
1301
1292
|
if not getattr(asset, "size", 0):
|
|
1302
1293
|
# size from HEAD header / Content-length
|
eodag/plugins/download/s3rest.py
CHANGED
|
@@ -20,7 +20,7 @@ from __future__ import annotations
|
|
|
20
20
|
import logging
|
|
21
21
|
import os
|
|
22
22
|
import os.path
|
|
23
|
-
from typing import TYPE_CHECKING, Dict, Optional, Union
|
|
23
|
+
from typing import TYPE_CHECKING, Dict, List, Optional, Union
|
|
24
24
|
from xml.dom import minidom
|
|
25
25
|
from xml.parsers.expat import ExpatError
|
|
26
26
|
|
|
@@ -68,7 +68,6 @@ class S3RestDownload(Download):
|
|
|
68
68
|
Re-use AwsDownload bucket some handling methods
|
|
69
69
|
|
|
70
70
|
:param provider: provider name
|
|
71
|
-
:type provider: str
|
|
72
71
|
:param config: Download plugin configuration:
|
|
73
72
|
|
|
74
73
|
* ``config.base_uri`` (str) - default endpoint url
|
|
@@ -80,8 +79,6 @@ class S3RestDownload(Download):
|
|
|
80
79
|
* ``config.order_headers`` (dict) - (optional) order request headers
|
|
81
80
|
* ``config.order_on_response`` (dict) - (optional) edit or add new product properties
|
|
82
81
|
* ``config.order_status`` (:class:`~eodag.config.PluginConfig.OrderStatus`) - Order status handling
|
|
83
|
-
|
|
84
|
-
:type config: :class:`~eodag.config.PluginConfig`
|
|
85
82
|
"""
|
|
86
83
|
|
|
87
84
|
def __init__(self, provider: str, config: PluginConfig) -> None:
|
|
@@ -100,22 +97,17 @@ class S3RestDownload(Download):
|
|
|
100
97
|
"""Download method for S3 REST API.
|
|
101
98
|
|
|
102
99
|
:param product: The EO product to download
|
|
103
|
-
:type product: :class:`~eodag.api.product._product.EOProduct`
|
|
104
100
|
:param auth: (optional) authenticated object
|
|
105
|
-
:type auth: Optional[Union[AuthBase, Dict[str, str]]]
|
|
106
101
|
:param progress_callback: (optional) A method or a callable object
|
|
107
102
|
which takes a current size and a maximum
|
|
108
103
|
size as inputs and handle progress bar
|
|
109
104
|
creation and update to give the user a
|
|
110
105
|
feedback on the download progress
|
|
111
|
-
:
|
|
112
|
-
:param kwargs: `outputs_prefix` (str), `extract` (bool), `delete_archive` (bool)
|
|
106
|
+
:param kwargs: `output_dir` (str), `extract` (bool), `delete_archive` (bool)
|
|
113
107
|
and `dl_url_params` (dict) can be provided as additional kwargs
|
|
114
108
|
and will override any other values defined in a configuration
|
|
115
109
|
file or with environment variables.
|
|
116
|
-
:type kwargs: Union[str, bool, dict]
|
|
117
110
|
:returns: The absolute path to the downloaded product in the local filesystem
|
|
118
|
-
:rtype: str
|
|
119
111
|
"""
|
|
120
112
|
if auth is not None and not isinstance(auth, AuthBase):
|
|
121
113
|
raise MisconfiguredError(f"Incompatible auth plugin: {type(auth)}")
|
|
@@ -133,7 +125,7 @@ class S3RestDownload(Download):
|
|
|
133
125
|
and "storageStatus" in product.properties
|
|
134
126
|
and product.properties["storageStatus"] != ONLINE_STATUS
|
|
135
127
|
):
|
|
136
|
-
self.http_download_plugin.
|
|
128
|
+
self.http_download_plugin.order_download(product=product, auth=auth)
|
|
137
129
|
|
|
138
130
|
@self._download_retry(product, wait, timeout)
|
|
139
131
|
def download_request(
|
|
@@ -145,7 +137,7 @@ class S3RestDownload(Download):
|
|
|
145
137
|
):
|
|
146
138
|
# check order status
|
|
147
139
|
if product.properties.get("orderStatusLink", None):
|
|
148
|
-
self.http_download_plugin.
|
|
140
|
+
self.http_download_plugin.order_download_status(
|
|
149
141
|
product=product, auth=auth
|
|
150
142
|
)
|
|
151
143
|
|
|
@@ -153,6 +145,8 @@ class S3RestDownload(Download):
|
|
|
153
145
|
bucket_name, prefix = get_bucket_name_and_prefix(
|
|
154
146
|
url=product.location, bucket_path_level=self.config.bucket_path_level
|
|
155
147
|
)
|
|
148
|
+
if prefix is None:
|
|
149
|
+
raise DownloadError(f"Could not extract prefix from {product.location}")
|
|
156
150
|
|
|
157
151
|
if (
|
|
158
152
|
bucket_name is None
|
|
@@ -243,14 +237,12 @@ class S3RestDownload(Download):
|
|
|
243
237
|
logger.warning("Could not load any content from %s", nodes_list_url)
|
|
244
238
|
|
|
245
239
|
# destination product path
|
|
246
|
-
|
|
247
|
-
|
|
248
|
-
)
|
|
249
|
-
abs_outputs_prefix = os.path.abspath(outputs_prefix)
|
|
250
|
-
product_local_path = os.path.join(abs_outputs_prefix, prefix.split("/")[-1])
|
|
240
|
+
output_dir = kwargs.pop("output_dir", None) or self.config.output_dir
|
|
241
|
+
abs_output_dir = os.path.abspath(output_dir)
|
|
242
|
+
product_local_path = os.path.join(abs_output_dir, prefix.split("/")[-1])
|
|
251
243
|
|
|
252
244
|
# .downloaded cache record directory
|
|
253
|
-
download_records_dir = os.path.join(
|
|
245
|
+
download_records_dir = os.path.join(abs_output_dir, ".downloaded")
|
|
254
246
|
try:
|
|
255
247
|
os.makedirs(download_records_dir)
|
|
256
248
|
except OSError as exc:
|
|
@@ -278,18 +270,18 @@ class S3RestDownload(Download):
|
|
|
278
270
|
os.remove(record_filename)
|
|
279
271
|
|
|
280
272
|
# total size for progress_callback
|
|
281
|
-
|
|
282
|
-
[
|
|
283
|
-
|
|
284
|
-
|
|
285
|
-
|
|
286
|
-
)
|
|
273
|
+
size_list: List[int] = [
|
|
274
|
+
int(node.firstChild.nodeValue) # type: ignore[attr-defined]
|
|
275
|
+
for node in xmldoc.getElementsByTagName("Size")
|
|
276
|
+
if node.firstChild is not None
|
|
277
|
+
]
|
|
278
|
+
total_size = sum(size_list)
|
|
287
279
|
progress_callback.reset(total=total_size)
|
|
288
280
|
|
|
289
281
|
# download each node key
|
|
290
282
|
for node_xml in nodes_xml_list:
|
|
291
283
|
node_key = unquote(
|
|
292
|
-
node_xml.getElementsByTagName("Key")[0].firstChild.nodeValue
|
|
284
|
+
node_xml.getElementsByTagName("Key")[0].firstChild.nodeValue # type: ignore[union-attr]
|
|
293
285
|
)
|
|
294
286
|
# As "Key", "Size" and "ETag" (md5 hash) can also be retrieved from node_xml
|
|
295
287
|
node_url = urljoin(bucket_url.strip("/") + "/", node_key.strip("/"))
|
eodag/plugins/manager.py
CHANGED
|
@@ -68,7 +68,6 @@ class PluginManager:
|
|
|
68
68
|
|
|
69
69
|
:param providers_config: The configuration with all information about the providers
|
|
70
70
|
supported by ``eodag``
|
|
71
|
-
:type providers_config: dict[str, :class:`~eodag.config.ProviderConfig`]
|
|
72
71
|
"""
|
|
73
72
|
|
|
74
73
|
supported_topics = {"search", "download", "crunch", "auth", "api"}
|
|
@@ -111,7 +110,11 @@ class PluginManager:
|
|
|
111
110
|
"Check that the plugin module (%s) is importable",
|
|
112
111
|
entry_point.module_name,
|
|
113
112
|
)
|
|
114
|
-
if
|
|
113
|
+
if (
|
|
114
|
+
entry_point.dist
|
|
115
|
+
and entry_point.dist.key != "eodag"
|
|
116
|
+
and entry_point.dist.location is not None
|
|
117
|
+
):
|
|
115
118
|
# use plugin providers if any
|
|
116
119
|
plugin_providers_config_path = [
|
|
117
120
|
str(x)
|
|
@@ -172,13 +175,10 @@ class PluginManager:
|
|
|
172
175
|
|
|
173
176
|
:param product_type: (optional) The product type that the constructed plugins
|
|
174
177
|
must support
|
|
175
|
-
:type product_type: str
|
|
176
178
|
:param provider: (optional) The provider or the provider group on which to get
|
|
177
179
|
the search plugins
|
|
178
|
-
:type provider: str
|
|
179
180
|
:returns: All the plugins supporting the product type, one by one (a generator
|
|
180
181
|
object)
|
|
181
|
-
:rtype: types.GeneratorType(:class:`~eodag.plugins.search.Search`
|
|
182
182
|
or :class:`~eodag.plugins.download.Api`)
|
|
183
183
|
:raises: :class:`~eodag.utils.exceptions.UnsupportedProvider`
|
|
184
184
|
"""
|
|
@@ -230,9 +230,7 @@ class PluginManager:
|
|
|
230
230
|
product.
|
|
231
231
|
|
|
232
232
|
:param product: The product to get a download plugin for
|
|
233
|
-
:type product: :class:`~eodag.api.product._product.EOProduct`
|
|
234
233
|
:returns: The download plugin capable of downloading the product
|
|
235
|
-
:rtype: :class:`~eodag.plugins.download.Download` or :class:`~eodag.plugins.download.Api`
|
|
236
234
|
"""
|
|
237
235
|
plugin_conf = self.providers_config[product.provider]
|
|
238
236
|
if download := getattr(plugin_conf, "download", None):
|
|
@@ -242,6 +240,7 @@ class PluginManager:
|
|
|
242
240
|
self._build_plugin(product.provider, download, Download),
|
|
243
241
|
)
|
|
244
242
|
elif api := getattr(plugin_conf, "api", None):
|
|
243
|
+
plugin_conf.api.products = plugin_conf.products
|
|
245
244
|
plugin_conf.api.priority = plugin_conf.priority
|
|
246
245
|
plugin = cast(Api, self._build_plugin(product.provider, api, Api))
|
|
247
246
|
else:
|
|
@@ -255,9 +254,7 @@ class PluginManager:
|
|
|
255
254
|
provider
|
|
256
255
|
|
|
257
256
|
:param provider: The provider for which to get the authentication plugin
|
|
258
|
-
:type provider: str
|
|
259
257
|
:returns: The Authentication plugin for the provider
|
|
260
|
-
:rtype: :class:`~eodag.plugins.authentication.Authentication`
|
|
261
258
|
"""
|
|
262
259
|
plugin_conf = self.providers_config[provider]
|
|
263
260
|
auth: Optional[PluginConfig] = getattr(plugin_conf, "auth", None)
|
|
@@ -278,11 +275,8 @@ class PluginManager:
|
|
|
278
275
|
it with the `options`
|
|
279
276
|
|
|
280
277
|
:param name: The name of the Crunch plugin to instantiate
|
|
281
|
-
:type name: str
|
|
282
278
|
:param options: The configuration parameters of the cruncher
|
|
283
|
-
:type options: dict
|
|
284
279
|
:returns: The cruncher named `name`
|
|
285
|
-
:rtype: :class:`~eodag.plugins.crunch.Crunch`
|
|
286
280
|
"""
|
|
287
281
|
klass = Crunch.get_plugin_by_class_name(name)
|
|
288
282
|
return klass(options)
|
|
@@ -296,9 +290,7 @@ class PluginManager:
|
|
|
296
290
|
"""Set the priority of the given provider
|
|
297
291
|
|
|
298
292
|
:param provider: The provider which is assigned the priority
|
|
299
|
-
:type provider: str
|
|
300
293
|
:param priority: The priority to assign to the provider
|
|
301
|
-
:type priority: int
|
|
302
294
|
"""
|
|
303
295
|
# Update the priority in the configurations so that it is taken into account
|
|
304
296
|
# when a plugin of this provider is latterly built
|
|
@@ -326,13 +318,9 @@ class PluginManager:
|
|
|
326
318
|
registered as the given provider
|
|
327
319
|
|
|
328
320
|
:param provider: The provider for which to build the plugin
|
|
329
|
-
:type provider: str
|
|
330
321
|
:param plugin_conf: The configuration of the plugin to be built
|
|
331
|
-
:type plugin_conf: :class:`~eodag.config.PluginConfig`
|
|
332
322
|
:param topic_class: The type of plugin to build
|
|
333
|
-
:type topic_class: :class:`~eodag.plugin.base.PluginTopic`
|
|
334
323
|
:returns: The built plugin
|
|
335
|
-
:rtype: :class:`~eodag.plugin.search.Search` or
|
|
336
324
|
:class:`~eodag.plugin.download.Download` or
|
|
337
325
|
:class:`~eodag.plugin.authentication.Authentication` or
|
|
338
326
|
:class:`~eodag.plugin.crunch.Crunch`
|
eodag/plugins/search/__init__.py
CHANGED
|
@@ -36,8 +36,8 @@ class PreparedSearch:
|
|
|
36
36
|
"""An object collecting needed information for search."""
|
|
37
37
|
|
|
38
38
|
product_type: Optional[str] = None
|
|
39
|
-
page: int = DEFAULT_PAGE
|
|
40
|
-
items_per_page: int = DEFAULT_ITEMS_PER_PAGE
|
|
39
|
+
page: Optional[int] = DEFAULT_PAGE
|
|
40
|
+
items_per_page: Optional[int] = DEFAULT_ITEMS_PER_PAGE
|
|
41
41
|
auth: Optional[Union[AuthBase, Dict[str, str]]] = None
|
|
42
42
|
auth_plugin: Optional[Authentication] = None
|
|
43
43
|
count: bool = True
|
|
@@ -45,10 +45,10 @@ class PreparedSearch:
|
|
|
45
45
|
info_message: Optional[str] = None
|
|
46
46
|
exception_message: Optional[str] = None
|
|
47
47
|
|
|
48
|
-
need_count: bool = field(init=False)
|
|
49
|
-
query_params: Dict[str, Any] = field(init=False)
|
|
50
|
-
query_string: str = field(init=False)
|
|
51
|
-
search_urls: List[str] = field(init=False)
|
|
52
|
-
product_type_def_params: Dict[str, Any] = field(init=False)
|
|
53
|
-
total_items_nb: int = field(init=False)
|
|
54
|
-
sort_by_qs: str = field(init=False)
|
|
48
|
+
need_count: bool = field(init=False, repr=False)
|
|
49
|
+
query_params: Dict[str, Any] = field(init=False, repr=False)
|
|
50
|
+
query_string: str = field(init=False, repr=False)
|
|
51
|
+
search_urls: List[str] = field(init=False, repr=False)
|
|
52
|
+
product_type_def_params: Dict[str, Any] = field(init=False, repr=False)
|
|
53
|
+
total_items_nb: int = field(init=False, repr=False)
|
|
54
|
+
sort_by_qs: str = field(init=False, repr=False)
|
eodag/plugins/search/base.py
CHANGED
|
@@ -59,9 +59,7 @@ class Search(PluginTopic):
|
|
|
59
59
|
"""Base Search Plugin.
|
|
60
60
|
|
|
61
61
|
:param provider: An EODAG provider name
|
|
62
|
-
:type provider: str
|
|
63
62
|
:param config: An EODAG plugin configuration
|
|
64
|
-
:type config: :class:`~eodag.config.PluginConfig`
|
|
65
63
|
"""
|
|
66
64
|
|
|
67
65
|
auth: Union[AuthBase, Dict[str, str]]
|
|
@@ -114,9 +112,7 @@ class Search(PluginTopic):
|
|
|
114
112
|
|
|
115
113
|
:param kwargs: additional filters for queryables (`productType` and other search
|
|
116
114
|
arguments)
|
|
117
|
-
:type kwargs: Any
|
|
118
115
|
:returns: fetched queryable parameters dict
|
|
119
|
-
:rtype: Optional[Dict[str, Annotated[Any, FieldInfo]]]
|
|
120
116
|
"""
|
|
121
117
|
raise NotImplementedError(
|
|
122
118
|
f"discover_queryables is not implemeted for plugin {self.__class__.__name__}"
|
|
@@ -129,9 +125,7 @@ class Search(PluginTopic):
|
|
|
129
125
|
Return given product type default settings as queryables
|
|
130
126
|
|
|
131
127
|
:param product_type: given product type
|
|
132
|
-
:type product_type: str
|
|
133
128
|
:returns: queryable parameters dict
|
|
134
|
-
:rtype: Dict[str, Annotated[Any, FieldInfo]]
|
|
135
129
|
"""
|
|
136
130
|
defaults = deepcopy(self.config.products.get(product_type, {}))
|
|
137
131
|
defaults.pop("metadata_mapping", None)
|
|
@@ -147,9 +141,7 @@ class Search(PluginTopic):
|
|
|
147
141
|
"""Get the provider product type from eodag product type
|
|
148
142
|
|
|
149
143
|
:param product_type: eodag product type
|
|
150
|
-
:type product_type: str
|
|
151
144
|
:returns: provider product type
|
|
152
|
-
:rtype: str
|
|
153
145
|
"""
|
|
154
146
|
if product_type is None:
|
|
155
147
|
return None
|
|
@@ -164,9 +156,7 @@ class Search(PluginTopic):
|
|
|
164
156
|
"""Get the provider product type definition parameters and specific settings
|
|
165
157
|
|
|
166
158
|
:param product_type: the desired product type
|
|
167
|
-
:type product_type: str
|
|
168
159
|
:returns: The product type definition parameters
|
|
169
|
-
:rtype: dict
|
|
170
160
|
"""
|
|
171
161
|
if product_type in self.config.products.keys():
|
|
172
162
|
logger.debug(
|
|
@@ -195,9 +185,7 @@ class Search(PluginTopic):
|
|
|
195
185
|
"""Get the plugin metadata mapping configuration (product type specific if exists)
|
|
196
186
|
|
|
197
187
|
:param product_type: the desired product type
|
|
198
|
-
:type product_type: str
|
|
199
188
|
:returns: The product type specific metadata-mapping
|
|
200
|
-
:rtype: dict
|
|
201
189
|
"""
|
|
202
190
|
if product_type:
|
|
203
191
|
return self.config.products.get(product_type, {}).get(
|
|
@@ -206,16 +194,14 @@ class Search(PluginTopic):
|
|
|
206
194
|
return self.config.metadata_mapping
|
|
207
195
|
|
|
208
196
|
def get_sort_by_arg(self, kwargs: Dict[str, Any]) -> Optional[SortByList]:
|
|
209
|
-
"""Extract the "
|
|
197
|
+
"""Extract the "sort_by" argument from the kwargs or the provider default sort configuration
|
|
210
198
|
|
|
211
199
|
:param kwargs: Search arguments
|
|
212
|
-
:
|
|
213
|
-
:returns: The "sortBy" argument from the kwargs or the provider default sort configuration
|
|
214
|
-
:rtype: :class:`~eodag.types.search_args.SortByList`
|
|
200
|
+
:returns: The "sort_by" argument from the kwargs or the provider default sort configuration
|
|
215
201
|
"""
|
|
216
|
-
# remove "
|
|
202
|
+
# remove "sort_by" from search args if exists because it is not part of metadata mapping,
|
|
217
203
|
# it will complete the query string or body once metadata mapping will be done
|
|
218
|
-
sort_by_arg_tmp = kwargs.pop("
|
|
204
|
+
sort_by_arg_tmp = kwargs.pop("sort_by", None)
|
|
219
205
|
sort_by_arg = sort_by_arg_tmp or getattr(self.config, "sort", {}).get(
|
|
220
206
|
"sort_by_default", None
|
|
221
207
|
)
|
|
@@ -230,12 +216,10 @@ class Search(PluginTopic):
|
|
|
230
216
|
self, sort_by_arg: SortByList
|
|
231
217
|
) -> Tuple[str, Dict[str, List[Dict[str, str]]]]:
|
|
232
218
|
"""Build the sorting part of the query string or body by transforming
|
|
233
|
-
the "
|
|
219
|
+
the "sort_by" argument into a provider-specific string or dictionnary
|
|
234
220
|
|
|
235
|
-
:param sort_by_arg: the "
|
|
236
|
-
:
|
|
237
|
-
:returns: The "sortBy" argument in provider-specific format
|
|
238
|
-
:rtype: Union[str, Dict[str, List[Dict[str, str]]]]
|
|
221
|
+
:param sort_by_arg: the "sort_by" argument in EODAG format
|
|
222
|
+
:returns: The "sort_by" argument in provider-specific format
|
|
239
223
|
"""
|
|
240
224
|
if not hasattr(self.config, "sort"):
|
|
241
225
|
raise ValidationError(f"{self.provider} does not support sorting feature")
|
|
@@ -331,13 +315,10 @@ class Search(PluginTopic):
|
|
|
331
315
|
Get queryables
|
|
332
316
|
|
|
333
317
|
:param filters: Additional filters for queryables.
|
|
334
|
-
:type filters: Dict[str, Any]
|
|
335
318
|
:param product_type: (optional) The product type.
|
|
336
|
-
:type product_type: Optional[str]
|
|
337
319
|
|
|
338
320
|
:return: A dictionary containing the queryable properties, associating parameters to their
|
|
339
321
|
annotated type.
|
|
340
|
-
:rtype: Dict[str, Annotated[Any, FieldInfo]]
|
|
341
322
|
"""
|
|
342
323
|
default_values: Dict[str, Any] = deepcopy(
|
|
343
324
|
getattr(self.config, "products", {}).get(product_type, {})
|