eodag 3.0.0b1__py3-none-any.whl → 3.0.0b3__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- eodag/__init__.py +6 -8
- eodag/api/core.py +119 -171
- eodag/api/product/__init__.py +10 -4
- eodag/api/product/_assets.py +52 -14
- eodag/api/product/_product.py +59 -30
- eodag/api/product/drivers/__init__.py +7 -2
- eodag/api/product/drivers/base.py +0 -3
- eodag/api/product/metadata_mapping.py +0 -28
- eodag/api/search_result.py +31 -9
- eodag/config.py +45 -41
- eodag/plugins/apis/base.py +3 -3
- eodag/plugins/apis/ecmwf.py +2 -3
- eodag/plugins/apis/usgs.py +43 -14
- eodag/plugins/authentication/aws_auth.py +11 -2
- eodag/plugins/authentication/openid_connect.py +5 -4
- eodag/plugins/authentication/token.py +2 -1
- eodag/plugins/crunch/base.py +3 -1
- eodag/plugins/crunch/filter_date.py +3 -9
- eodag/plugins/crunch/filter_latest_intersect.py +0 -3
- eodag/plugins/crunch/filter_latest_tpl_name.py +1 -4
- eodag/plugins/crunch/filter_overlap.py +4 -8
- eodag/plugins/crunch/filter_property.py +5 -11
- eodag/plugins/download/aws.py +46 -78
- eodag/plugins/download/base.py +27 -68
- eodag/plugins/download/http.py +48 -57
- eodag/plugins/download/s3rest.py +17 -25
- eodag/plugins/manager.py +6 -18
- eodag/plugins/search/__init__.py +9 -9
- eodag/plugins/search/base.py +7 -26
- eodag/plugins/search/build_search_result.py +0 -13
- eodag/plugins/search/cop_marine.py +1 -3
- eodag/plugins/search/creodias_s3.py +0 -3
- eodag/plugins/search/data_request_search.py +10 -5
- eodag/plugins/search/qssearch.py +95 -53
- eodag/plugins/search/static_stac_search.py +6 -3
- eodag/resources/ext_product_types.json +1 -1
- eodag/resources/product_types.yml +24 -0
- eodag/resources/providers.yml +198 -154
- eodag/resources/user_conf_template.yml +27 -27
- eodag/rest/core.py +11 -43
- eodag/rest/server.py +1 -6
- eodag/rest/stac.py +13 -87
- eodag/rest/types/eodag_search.py +4 -7
- eodag/rest/types/queryables.py +4 -12
- eodag/rest/types/stac_search.py +7 -11
- eodag/rest/utils/rfc3339.py +0 -1
- eodag/types/__init__.py +9 -3
- eodag/types/download_args.py +14 -5
- eodag/types/search_args.py +7 -8
- eodag/types/whoosh.py +0 -2
- eodag/utils/__init__.py +20 -79
- eodag/utils/constraints.py +0 -8
- eodag/utils/import_system.py +0 -4
- eodag/utils/logging.py +0 -3
- eodag/utils/notebook.py +4 -4
- eodag/utils/repr.py +113 -0
- eodag/utils/requests.py +12 -20
- eodag/utils/rest.py +0 -4
- eodag/utils/stac_reader.py +2 -14
- {eodag-3.0.0b1.dist-info → eodag-3.0.0b3.dist-info}/METADATA +33 -14
- eodag-3.0.0b3.dist-info/RECORD +110 -0
- {eodag-3.0.0b1.dist-info → eodag-3.0.0b3.dist-info}/WHEEL +1 -1
- eodag-3.0.0b1.dist-info/RECORD +0 -109
- {eodag-3.0.0b1.dist-info → eodag-3.0.0b3.dist-info}/LICENSE +0 -0
- {eodag-3.0.0b1.dist-info → eodag-3.0.0b3.dist-info}/entry_points.txt +0 -0
- {eodag-3.0.0b1.dist-info → eodag-3.0.0b3.dist-info}/top_level.txt +0 -0
|
@@ -19,7 +19,7 @@ from __future__ import annotations
|
|
|
19
19
|
|
|
20
20
|
import logging
|
|
21
21
|
import operator
|
|
22
|
-
from typing import TYPE_CHECKING, Any,
|
|
22
|
+
from typing import TYPE_CHECKING, Any, List
|
|
23
23
|
|
|
24
24
|
from eodag.plugins.crunch.base import Crunch
|
|
25
25
|
|
|
@@ -38,23 +38,17 @@ class FilterProperty(Crunch):
|
|
|
38
38
|
|
|
39
39
|
- `property=value` : property key from product.properties, associated to its filter value
|
|
40
40
|
- `operator` : (optional) Operator used for filtering (one of `lt,le,eq,ne,ge,gt`). Default is `eq`
|
|
41
|
-
|
|
42
|
-
:type config: dict
|
|
43
41
|
"""
|
|
44
42
|
|
|
45
|
-
config: Dict[str, Union[str, Optional[str]]]
|
|
46
|
-
|
|
47
43
|
def proceed(
|
|
48
44
|
self, products: List[EOProduct], **search_params: Any
|
|
49
45
|
) -> List[EOProduct]:
|
|
50
46
|
"""Execute crunch: Filter products, retaining only those that match property filtering
|
|
51
47
|
|
|
52
48
|
:param products: A list of products resulting from a search
|
|
53
|
-
:type products: list(:class:`~eodag.api.product._product.EOProduct`)
|
|
54
49
|
:returns: The filtered products
|
|
55
|
-
:rtype: list(:class:`~eodag.api.product._product.EOProduct`)
|
|
56
50
|
"""
|
|
57
|
-
operator_name = self.config.pop("operator", "eq") or "eq"
|
|
51
|
+
operator_name = self.config.__dict__.pop("operator", "eq") or "eq"
|
|
58
52
|
try:
|
|
59
53
|
operator_method = getattr(operator, operator_name)
|
|
60
54
|
except AttributeError:
|
|
@@ -64,12 +58,12 @@ class FilterProperty(Crunch):
|
|
|
64
58
|
)
|
|
65
59
|
return products
|
|
66
60
|
|
|
67
|
-
if len(self.config.keys()) != 1:
|
|
61
|
+
if len(self.config.__dict__.keys()) != 1:
|
|
68
62
|
logger.warning("One property is needed for filtering, filtering disabled.")
|
|
69
63
|
return products
|
|
70
64
|
|
|
71
|
-
property_key = next(iter(self.config))
|
|
72
|
-
property_value = self.config.get(property_key, None)
|
|
65
|
+
property_key = next(iter(self.config.__dict__))
|
|
66
|
+
property_value = self.config.__dict__.get(property_key, None)
|
|
73
67
|
|
|
74
68
|
logger.debug(
|
|
75
69
|
"Start filtering for products matching operator.%s(product.properties['%s'], %s)",
|
eodag/plugins/download/aws.py
CHANGED
|
@@ -34,6 +34,7 @@ from typing import (
|
|
|
34
34
|
Optional,
|
|
35
35
|
Set,
|
|
36
36
|
Tuple,
|
|
37
|
+
TypedDict,
|
|
37
38
|
Union,
|
|
38
39
|
cast,
|
|
39
40
|
)
|
|
@@ -213,7 +214,6 @@ class AwsDownload(Download):
|
|
|
213
214
|
"""Download on AWS using S3 protocol.
|
|
214
215
|
|
|
215
216
|
:param provider: provider name
|
|
216
|
-
:type provider: str
|
|
217
217
|
:param config: Download plugin configuration:
|
|
218
218
|
|
|
219
219
|
* ``config.base_uri`` (str) - s3 endpoint url
|
|
@@ -222,14 +222,12 @@ class AwsDownload(Download):
|
|
|
222
222
|
* ``config.flatten_top_dirs`` (bool) - (optional) flatten directory structure
|
|
223
223
|
* ``config.products`` (dict) - (optional) product_type specific configuration
|
|
224
224
|
* ``config.ignore_assets`` (bool) - (optional) ignore assets and download using downloadLink
|
|
225
|
-
|
|
226
|
-
:type config: :class:`~eodag.config.PluginConfig`
|
|
227
225
|
"""
|
|
228
226
|
|
|
229
227
|
def __init__(self, provider: str, config: PluginConfig) -> None:
|
|
230
228
|
super(AwsDownload, self).__init__(provider, config)
|
|
231
229
|
self.requester_pays = getattr(self.config, "requester_pays", False)
|
|
232
|
-
self.s3_session = None
|
|
230
|
+
self.s3_session: Optional[boto3.session.Session] = None
|
|
233
231
|
|
|
234
232
|
def download(
|
|
235
233
|
self,
|
|
@@ -246,25 +244,20 @@ class AwsDownload(Download):
|
|
|
246
244
|
SAFE-build is configured for a given provider and product type.
|
|
247
245
|
If the product title is configured to be updated during download and
|
|
248
246
|
SAFE-formatted, its destination path will be:
|
|
249
|
-
`{
|
|
247
|
+
`{output_dir}/{title}`
|
|
250
248
|
|
|
251
249
|
:param product: The EO product to download
|
|
252
|
-
:type product: :class:`~eodag.api.product._product.EOProduct`
|
|
253
250
|
:param auth: (optional) authenticated object
|
|
254
|
-
:type auth: Union[AuthBase, Dict[str, str]]
|
|
255
251
|
:param progress_callback: (optional) A method or a callable object
|
|
256
252
|
which takes a current size and a maximum
|
|
257
253
|
size as inputs and handle progress bar
|
|
258
254
|
creation and update to give the user a
|
|
259
255
|
feedback on the download progress
|
|
260
|
-
:
|
|
261
|
-
:param kwargs: `outputs_prefix` (str), `extract` (bool), `delete_archive` (bool)
|
|
256
|
+
:param kwargs: `output_dir` (str), `extract` (bool), `delete_archive` (bool)
|
|
262
257
|
and `dl_url_params` (dict) can be provided as additional kwargs
|
|
263
258
|
and will override any other values defined in a configuration
|
|
264
259
|
file or with environment variables.
|
|
265
|
-
:type kwargs: Union[str, bool, dict]
|
|
266
260
|
:returns: The absolute path to the downloaded product in the local filesystem
|
|
267
|
-
:rtype: str
|
|
268
261
|
"""
|
|
269
262
|
if auth is None:
|
|
270
263
|
auth = {}
|
|
@@ -404,13 +397,9 @@ class AwsDownload(Download):
|
|
|
404
397
|
- get file path
|
|
405
398
|
- create directories
|
|
406
399
|
:param product: product to be downloaded
|
|
407
|
-
:type product: EOProduct
|
|
408
400
|
:param progress_callback: progress callback to be used
|
|
409
|
-
:type progress_callback: ProgressCallback
|
|
410
401
|
:param kwargs: additional arguments
|
|
411
|
-
:type kwargs: Any
|
|
412
402
|
:return: local path and file name
|
|
413
|
-
:rtype: Tuple[str, Optional[str]]
|
|
414
403
|
"""
|
|
415
404
|
product_local_path, record_filename = self._prepare_download(
|
|
416
405
|
product, progress_callback=progress_callback, **kwargs
|
|
@@ -432,9 +421,7 @@ class AwsDownload(Download):
|
|
|
432
421
|
"""
|
|
433
422
|
updates the product properties with fetch metadata if safe build is enabled
|
|
434
423
|
:param build_safe: if safe build is enabled
|
|
435
|
-
:type build_safe: bool
|
|
436
424
|
:param product: product to be updated
|
|
437
|
-
:type product: EOProduct
|
|
438
425
|
"""
|
|
439
426
|
product_conf = getattr(self.config, "products", {}).get(
|
|
440
427
|
product.product_type, {}
|
|
@@ -480,13 +467,9 @@ class AwsDownload(Download):
|
|
|
480
467
|
"""
|
|
481
468
|
retrieves the bucket names and path prefixes for the assets
|
|
482
469
|
:param product: product for which the assets shall be downloaded
|
|
483
|
-
:type product: EOProduct
|
|
484
470
|
:param asset_filter: text for which the assets should be filtered
|
|
485
|
-
:type asset_filter: str
|
|
486
471
|
:param ignore_assets: if product instead of individual assets should be used
|
|
487
|
-
:type ignore_assets: bool
|
|
488
472
|
:return: tuples of bucket names and prefixes
|
|
489
|
-
:rtype: List[Tuple[str, Optional[str]]]
|
|
490
473
|
"""
|
|
491
474
|
# if assets are defined, use them instead of scanning product.location
|
|
492
475
|
if len(product.assets) > 0 and not ignore_assets:
|
|
@@ -528,11 +511,8 @@ class AwsDownload(Download):
|
|
|
528
511
|
authenticates with s3 and retrieves the available objects
|
|
529
512
|
raises an error when authentication is not possible
|
|
530
513
|
:param bucket_names_and_prefixes: list of bucket names and corresponding path prefixes
|
|
531
|
-
:type bucket_names_and_prefixes: List[Tuple[str, Optional[str]]]
|
|
532
514
|
:param auth: authentication information
|
|
533
|
-
:type auth: Dict[str, str]
|
|
534
515
|
:return: authenticated objects per bucket, list of available objects
|
|
535
|
-
:rtype: Tuple[Dict[str, Any], ResourceCollection[Any]]
|
|
536
516
|
"""
|
|
537
517
|
if not isinstance(auth, (dict, type(None))):
|
|
538
518
|
raise AuthenticationError(
|
|
@@ -602,17 +582,11 @@ class AwsDownload(Download):
|
|
|
602
582
|
"""
|
|
603
583
|
retrieve unique product chunks based on authenticated objects and asset filters
|
|
604
584
|
:param bucket_names_and_prefixes: list of bucket names and corresponding path prefixes
|
|
605
|
-
:type bucket_names_and_prefixes: List[Tuple[str, Optional[str]]]
|
|
606
585
|
:param authenticated_objects: available objects per bucket
|
|
607
|
-
:type authenticated_objects: Dict[str, Any]
|
|
608
586
|
:param asset_filter: text for which assets should be filtered
|
|
609
|
-
:type asset_filter: str
|
|
610
587
|
:param ignore_assets: if product instead of individual assets should be used
|
|
611
|
-
:type ignore_assets: bool
|
|
612
588
|
:param product: product that shall be downloaded
|
|
613
|
-
:type product: EOProduct
|
|
614
589
|
:return: set of product chunks that can be downloaded
|
|
615
|
-
:rtype: Set[Any]
|
|
616
590
|
"""
|
|
617
591
|
product_chunks: List[Any] = []
|
|
618
592
|
for bucket_name, prefix in bucket_names_and_prefixes:
|
|
@@ -666,23 +640,16 @@ class AwsDownload(Download):
|
|
|
666
640
|
It contains a generator to streamed download chunks and the response headers.
|
|
667
641
|
|
|
668
642
|
:param product: The EO product to download
|
|
669
|
-
:type product: :class:`~eodag.api.product._product.EOProduct`
|
|
670
643
|
:param auth: (optional) The configuration of a plugin of type Authentication
|
|
671
|
-
:type auth: :class:`~eodag.config.PluginConfig`
|
|
672
644
|
:param progress_callback: (optional) A progress callback
|
|
673
|
-
:type progress_callback: :class:`~eodag.utils.ProgressCallback`
|
|
674
645
|
:param wait: (optional) If download fails, wait time in minutes between two download tries
|
|
675
|
-
:type wait: int
|
|
676
646
|
:param timeout: (optional) If download fails, maximum time in minutes before stop retrying
|
|
677
647
|
to download
|
|
678
|
-
:
|
|
679
|
-
:param kwargs: `outputs_prefix` (str), `extract` (bool), `delete_archive` (bool)
|
|
648
|
+
:param kwargs: `output_dir` (str), `extract` (bool), `delete_archive` (bool)
|
|
680
649
|
and `dl_url_params` (dict) can be provided as additional kwargs
|
|
681
650
|
and will override any other values defined in a configuration
|
|
682
651
|
file or with environment variables.
|
|
683
|
-
:type kwargs: Union[str, bool, dict]
|
|
684
652
|
:returns: Dictionnary of :class:`~fastapi.responses.StreamingResponse` keyword-arguments
|
|
685
|
-
:rtype: dict
|
|
686
653
|
"""
|
|
687
654
|
if progress_callback is None:
|
|
688
655
|
logger.info(
|
|
@@ -773,7 +740,7 @@ class AwsDownload(Download):
|
|
|
773
740
|
build_safe: bool,
|
|
774
741
|
progress_callback: ProgressCallback,
|
|
775
742
|
assets_values: List[Dict[str, Any]],
|
|
776
|
-
) -> Iterator[
|
|
743
|
+
) -> Iterator[Any]:
|
|
777
744
|
"""Yield product data chunks"""
|
|
778
745
|
|
|
779
746
|
chunk_size = 4096 * 1024
|
|
@@ -861,13 +828,9 @@ class AwsDownload(Download):
|
|
|
861
828
|
"""Get rasterio environment variables needed for data access authentication.
|
|
862
829
|
|
|
863
830
|
:param bucket_name: Bucket containg objects
|
|
864
|
-
:type bucket_name: str
|
|
865
831
|
:param prefix: Prefix used to try auth
|
|
866
|
-
:type prefix: str
|
|
867
832
|
:param auth_dict: Dictionnary containing authentication keys
|
|
868
|
-
:type auth_dict: dict
|
|
869
833
|
:returns: The rasterio environement variables
|
|
870
|
-
:rtype: dict
|
|
871
834
|
"""
|
|
872
835
|
if self.s3_session is not None:
|
|
873
836
|
if self.requester_pays:
|
|
@@ -892,14 +855,10 @@ class AwsDownload(Download):
|
|
|
892
855
|
Also expose ``s3_session`` as class variable if available.
|
|
893
856
|
|
|
894
857
|
:param bucket_name: Bucket containg objects
|
|
895
|
-
:type bucket_name: str
|
|
896
858
|
:param prefix: Prefix used to filter objects on auth try
|
|
897
859
|
(not used to filter returned objects)
|
|
898
|
-
:type prefix: str
|
|
899
860
|
:param auth_dict: Dictionnary containing authentication keys
|
|
900
|
-
:type auth_dict: dict
|
|
901
861
|
:returns: The boto3 authenticated objects
|
|
902
|
-
:rtype: :class:`~boto3.resources.collection.s3.Bucket.objectsCollection`
|
|
903
862
|
"""
|
|
904
863
|
auth_methods: List[
|
|
905
864
|
Callable[[str, str, Dict[str, str]], Optional[ResourceCollection]]
|
|
@@ -941,15 +900,15 @@ class AwsDownload(Download):
|
|
|
941
900
|
) -> Optional[ResourceCollection]:
|
|
942
901
|
"""Auth strategy using no-sign-request"""
|
|
943
902
|
|
|
944
|
-
s3_resource = boto3.resource(
|
|
903
|
+
s3_resource = boto3.resource(
|
|
945
904
|
service_name="s3", endpoint_url=getattr(self.config, "base_uri", None)
|
|
946
905
|
)
|
|
947
|
-
s3_resource.meta.client.meta.events.register(
|
|
906
|
+
s3_resource.meta.client.meta.events.register(
|
|
948
907
|
"choose-signer.s3.*", disable_signing
|
|
949
908
|
)
|
|
950
|
-
objects = s3_resource.Bucket(bucket_name).objects
|
|
951
|
-
list(objects.filter(Prefix=prefix).limit(1))
|
|
952
|
-
return objects
|
|
909
|
+
objects = s3_resource.Bucket(bucket_name).objects
|
|
910
|
+
list(objects.filter(Prefix=prefix).limit(1))
|
|
911
|
+
return objects
|
|
953
912
|
|
|
954
913
|
def _get_authenticated_objects_from_auth_profile(
|
|
955
914
|
self, bucket_name: str, prefix: str, auth_dict: Dict[str, str]
|
|
@@ -957,20 +916,20 @@ class AwsDownload(Download):
|
|
|
957
916
|
"""Auth strategy using RequestPayer=requester and ``aws_profile`` from provided credentials"""
|
|
958
917
|
|
|
959
918
|
if "profile_name" in auth_dict.keys():
|
|
960
|
-
s3_session = boto3.session.Session(profile_name=auth_dict["profile_name"])
|
|
961
|
-
s3_resource = s3_session.resource(
|
|
919
|
+
s3_session = boto3.session.Session(profile_name=auth_dict["profile_name"])
|
|
920
|
+
s3_resource = s3_session.resource(
|
|
962
921
|
service_name="s3",
|
|
963
922
|
endpoint_url=getattr(self.config, "base_uri", None),
|
|
964
923
|
)
|
|
965
924
|
if self.requester_pays:
|
|
966
|
-
objects = s3_resource.Bucket(bucket_name).objects.filter(
|
|
925
|
+
objects = s3_resource.Bucket(bucket_name).objects.filter(
|
|
967
926
|
RequestPayer="requester"
|
|
968
927
|
)
|
|
969
928
|
else:
|
|
970
|
-
objects = s3_resource.Bucket(bucket_name).objects
|
|
971
|
-
list(objects.filter(Prefix=prefix).limit(1))
|
|
972
|
-
self.s3_session = s3_session
|
|
973
|
-
return objects
|
|
929
|
+
objects = s3_resource.Bucket(bucket_name).objects
|
|
930
|
+
list(objects.filter(Prefix=prefix).limit(1))
|
|
931
|
+
self.s3_session = s3_session
|
|
932
|
+
return objects
|
|
974
933
|
else:
|
|
975
934
|
return None
|
|
976
935
|
|
|
@@ -981,23 +940,35 @@ class AwsDownload(Download):
|
|
|
981
940
|
from provided credentials"""
|
|
982
941
|
|
|
983
942
|
if all(k in auth_dict for k in ("aws_access_key_id", "aws_secret_access_key")):
|
|
984
|
-
|
|
985
|
-
|
|
986
|
-
|
|
943
|
+
S3SessionKwargs = TypedDict(
|
|
944
|
+
"S3SessionKwargs",
|
|
945
|
+
{
|
|
946
|
+
"aws_access_key_id": str,
|
|
947
|
+
"aws_secret_access_key": str,
|
|
948
|
+
"aws_session_token": str,
|
|
949
|
+
},
|
|
950
|
+
total=False,
|
|
987
951
|
)
|
|
988
|
-
|
|
952
|
+
s3_session_kwargs: S3SessionKwargs = {
|
|
953
|
+
"aws_access_key_id": auth_dict["aws_access_key_id"],
|
|
954
|
+
"aws_secret_access_key": auth_dict["aws_secret_access_key"],
|
|
955
|
+
}
|
|
956
|
+
if auth_dict.get("aws_session_token"):
|
|
957
|
+
s3_session_kwargs["aws_session_token"] = auth_dict["aws_session_token"]
|
|
958
|
+
s3_session = boto3.session.Session(**s3_session_kwargs)
|
|
959
|
+
s3_resource = s3_session.resource(
|
|
989
960
|
service_name="s3",
|
|
990
961
|
endpoint_url=getattr(self.config, "base_uri", None),
|
|
991
962
|
)
|
|
992
963
|
if self.requester_pays:
|
|
993
|
-
objects = s3_resource.Bucket(bucket_name).objects.filter(
|
|
964
|
+
objects = s3_resource.Bucket(bucket_name).objects.filter(
|
|
994
965
|
RequestPayer="requester"
|
|
995
966
|
)
|
|
996
967
|
else:
|
|
997
|
-
objects = s3_resource.Bucket(bucket_name).objects
|
|
998
|
-
list(objects.filter(Prefix=prefix).limit(1))
|
|
999
|
-
self.s3_session = s3_session
|
|
1000
|
-
return objects
|
|
968
|
+
objects = s3_resource.Bucket(bucket_name).objects
|
|
969
|
+
list(objects.filter(Prefix=prefix).limit(1))
|
|
970
|
+
self.s3_session = s3_session
|
|
971
|
+
return objects
|
|
1001
972
|
else:
|
|
1002
973
|
return None
|
|
1003
974
|
|
|
@@ -1006,19 +977,19 @@ class AwsDownload(Download):
|
|
|
1006
977
|
) -> Optional[ResourceCollection]:
|
|
1007
978
|
"""Auth strategy using RequestPayer=requester and current environment"""
|
|
1008
979
|
|
|
1009
|
-
s3_session = boto3.session.Session()
|
|
1010
|
-
s3_resource = s3_session.resource(
|
|
980
|
+
s3_session = boto3.session.Session()
|
|
981
|
+
s3_resource = s3_session.resource(
|
|
1011
982
|
service_name="s3", endpoint_url=getattr(self.config, "base_uri", None)
|
|
1012
983
|
)
|
|
1013
984
|
if self.requester_pays:
|
|
1014
|
-
objects = s3_resource.Bucket(bucket_name).objects.filter(
|
|
985
|
+
objects = s3_resource.Bucket(bucket_name).objects.filter(
|
|
1015
986
|
RequestPayer="requester"
|
|
1016
987
|
)
|
|
1017
988
|
else:
|
|
1018
|
-
objects = s3_resource.Bucket(bucket_name).objects
|
|
1019
|
-
list(objects.filter(Prefix=prefix).limit(1))
|
|
1020
|
-
self.s3_session = s3_session
|
|
1021
|
-
return objects
|
|
989
|
+
objects = s3_resource.Bucket(bucket_name).objects
|
|
990
|
+
list(objects.filter(Prefix=prefix).limit(1))
|
|
991
|
+
self.s3_session = s3_session
|
|
992
|
+
return objects
|
|
1022
993
|
|
|
1023
994
|
def get_product_bucket_name_and_prefix(
|
|
1024
995
|
self, product: EOProduct, url: Optional[str] = None
|
|
@@ -1026,11 +997,8 @@ class AwsDownload(Download):
|
|
|
1026
997
|
"""Extract bucket name and prefix from product URL
|
|
1027
998
|
|
|
1028
999
|
:param product: The EO product to download
|
|
1029
|
-
:type product: :class:`~eodag.api.product._product.EOProduct`
|
|
1030
1000
|
:param url: (optional) URL to use as product.location
|
|
1031
|
-
:type url: str
|
|
1032
1001
|
:returns: bucket_name and prefix as str
|
|
1033
|
-
:rtype: tuple
|
|
1034
1002
|
"""
|
|
1035
1003
|
if url is None:
|
|
1036
1004
|
url = product.location
|
eodag/plugins/download/base.py
CHANGED
|
@@ -79,17 +79,17 @@ class Download(PluginTopic):
|
|
|
79
79
|
|
|
80
80
|
They must:
|
|
81
81
|
|
|
82
|
-
- download data in the ``
|
|
82
|
+
- download data in the ``output_dir`` folder defined in the plugin's
|
|
83
83
|
configuration or passed through kwargs
|
|
84
84
|
- extract products from their archive (if relevant) if ``extract`` is set to True
|
|
85
85
|
(True by default)
|
|
86
|
-
- save a product in an archive/directory (in ``
|
|
86
|
+
- save a product in an archive/directory (in ``output_dir``) whose name must be
|
|
87
87
|
the product's ``title`` property
|
|
88
88
|
- update the product's ``location`` attribute once its data is downloaded (and
|
|
89
89
|
eventually after it's extracted) to the product's location given as a file URI
|
|
90
90
|
(e.g. 'file:///tmp/product_folder' on Linux or
|
|
91
91
|
'file:///C:/Users/username/AppData/LOcal/Temp' on Windows)
|
|
92
|
-
- save a *record* file in the directory ``
|
|
92
|
+
- save a *record* file in the directory ``output_dir/.downloaded`` whose name
|
|
93
93
|
is built on the MD5 hash of the product's ``product_type`` and ``properties['id']``
|
|
94
94
|
attributes (``hashlib.md5((product.product_type+"-"+product.properties['id']).encode("utf-8")).hexdigest()``)
|
|
95
95
|
and whose content is the product's ``remote_location`` attribute itself.
|
|
@@ -100,9 +100,7 @@ class Download(PluginTopic):
|
|
|
100
100
|
(it certainly indicates that the download didn't complete)
|
|
101
101
|
|
|
102
102
|
:param provider: An eodag providers configuration dictionary
|
|
103
|
-
:type provider: dict
|
|
104
103
|
:param config: Path to the user configuration file
|
|
105
|
-
:type config: str
|
|
106
104
|
"""
|
|
107
105
|
|
|
108
106
|
def __init__(self, provider: str, config: PluginConfig) -> None:
|
|
@@ -122,25 +120,18 @@ class Download(PluginTopic):
|
|
|
122
120
|
Base download method. Not available, it must be defined for each plugin.
|
|
123
121
|
|
|
124
122
|
:param product: The EO product to download
|
|
125
|
-
:type product: :class:`~eodag.api.product._product.EOProduct`
|
|
126
123
|
:param auth: (optional) authenticated object
|
|
127
|
-
:type auth: Optional[Union[AuthBase, Dict[str, str]]]
|
|
128
124
|
:param progress_callback: (optional) A progress callback
|
|
129
|
-
:type progress_callback: :class:`~eodag.utils.ProgressCallback`
|
|
130
125
|
:param wait: (optional) If download fails, wait time in minutes between two download tries
|
|
131
|
-
:type wait: int
|
|
132
126
|
:param timeout: (optional) If download fails, maximum time in minutes before stop retrying
|
|
133
127
|
to download
|
|
134
|
-
:
|
|
135
|
-
:param kwargs: `outputs_prefix` (str), `extract` (bool), `delete_archive` (bool)
|
|
128
|
+
:param kwargs: `output_dir` (str), `extract` (bool), `delete_archive` (bool)
|
|
136
129
|
and `dl_url_params` (dict) can be provided as additional kwargs
|
|
137
130
|
and will override any other values defined in a configuration
|
|
138
131
|
file or with environment variables.
|
|
139
|
-
:type kwargs: Union[str, bool, dict]
|
|
140
132
|
:returns: The absolute path to the downloaded product in the local filesystem
|
|
141
133
|
(e.g. '/tmp/product.zip' on Linux or
|
|
142
134
|
'C:\\Users\\username\\AppData\\Local\\Temp\\product.zip' on Windows)
|
|
143
|
-
:rtype: str
|
|
144
135
|
"""
|
|
145
136
|
raise NotImplementedError(
|
|
146
137
|
"A Download plugin must implement a method named download"
|
|
@@ -159,23 +150,16 @@ class Download(PluginTopic):
|
|
|
159
150
|
Base _stream_download_dict method. Not available, it must be defined for each plugin.
|
|
160
151
|
|
|
161
152
|
:param product: The EO product to download
|
|
162
|
-
:type product: :class:`~eodag.api.product._product.EOProduct`
|
|
163
153
|
:param auth: (optional) authenticated object
|
|
164
|
-
:type auth: Optional[Union[AuthBase, Dict[str, str]]]
|
|
165
154
|
:param progress_callback: (optional) A progress callback
|
|
166
|
-
:type progress_callback: :class:`~eodag.utils.ProgressCallback`
|
|
167
155
|
:param wait: (optional) If download fails, wait time in minutes between two download tries
|
|
168
|
-
:type wait: int
|
|
169
156
|
:param timeout: (optional) If download fails, maximum time in minutes before stop retrying
|
|
170
157
|
to download
|
|
171
|
-
:
|
|
172
|
-
:param kwargs: `outputs_prefix` (str), `extract` (bool), `delete_archive` (bool)
|
|
158
|
+
:param kwargs: `output_dir` (str), `extract` (bool), `delete_archive` (bool)
|
|
173
159
|
and `dl_url_params` (dict) can be provided as additional kwargs
|
|
174
160
|
and will override any other values defined in a configuration
|
|
175
161
|
file or with environment variables.
|
|
176
|
-
:type kwargs: Union[str, bool, dict]
|
|
177
162
|
:returns: Dictionnary of :class:`~fastapi.responses.StreamingResponse` keyword-arguments
|
|
178
|
-
:rtype: dict
|
|
179
163
|
"""
|
|
180
164
|
raise NotImplementedError(
|
|
181
165
|
"Download streaming must be implemented using a method named _stream_download_dict"
|
|
@@ -190,11 +174,8 @@ class Download(PluginTopic):
|
|
|
190
174
|
"""Check if file has already been downloaded, and prepare product download
|
|
191
175
|
|
|
192
176
|
:param product: The EO product to download
|
|
193
|
-
:type product: :class:`~eodag.api.product._product.EOProduct`
|
|
194
177
|
:param progress_callback: (optional) A progress callback
|
|
195
|
-
:type progress_callback: :class:`~eodag.utils.ProgressCallback` or None
|
|
196
178
|
:returns: fs_path, record_filename
|
|
197
|
-
:rtype: Tuple[Optional[str], Optional[str]]
|
|
198
179
|
"""
|
|
199
180
|
if product.location != product.remote_location:
|
|
200
181
|
fs_path = uri_to_path(product.location)
|
|
@@ -216,18 +197,18 @@ class Download(PluginTopic):
|
|
|
216
197
|
f"Download url: {url}",
|
|
217
198
|
)
|
|
218
199
|
|
|
219
|
-
|
|
220
|
-
kwargs.pop("
|
|
221
|
-
or getattr(self.config, "
|
|
200
|
+
output_dir = (
|
|
201
|
+
kwargs.pop("output_dir", None)
|
|
202
|
+
or getattr(self.config, "output_dir", tempfile.gettempdir())
|
|
222
203
|
or tempfile.gettempdir()
|
|
223
204
|
)
|
|
224
|
-
|
|
225
|
-
self.config, "
|
|
205
|
+
output_extension = kwargs.get("output_extension", None) or getattr(
|
|
206
|
+
self.config, "output_extension", ".zip"
|
|
226
207
|
)
|
|
227
208
|
|
|
228
209
|
# Strong asumption made here: all products downloaded will be zip files
|
|
229
210
|
# If they are not, the '.zip' extension will be removed when they are downloaded and returned as is
|
|
230
|
-
prefix = os.path.abspath(
|
|
211
|
+
prefix = os.path.abspath(output_dir)
|
|
231
212
|
sanitized_title = sanitize(product.properties["title"])
|
|
232
213
|
if sanitized_title == product.properties["title"]:
|
|
233
214
|
collision_avoidance_suffix = ""
|
|
@@ -235,10 +216,10 @@ class Download(PluginTopic):
|
|
|
235
216
|
collision_avoidance_suffix = "-" + sanitize(product.properties["id"])
|
|
236
217
|
fs_path = os.path.join(
|
|
237
218
|
prefix,
|
|
238
|
-
f"{sanitize(product.properties['title'])}{collision_avoidance_suffix}{
|
|
219
|
+
f"{sanitize(product.properties['title'])}{collision_avoidance_suffix}{output_extension}",
|
|
239
220
|
)
|
|
240
221
|
fs_dir_path = (
|
|
241
|
-
fs_path.replace(
|
|
222
|
+
fs_path.replace(output_extension, "") if output_extension else fs_path
|
|
242
223
|
)
|
|
243
224
|
download_records_dir = os.path.join(prefix, ".downloaded")
|
|
244
225
|
try:
|
|
@@ -292,9 +273,7 @@ class Download(PluginTopic):
|
|
|
292
273
|
(``hashlib.md5((product.product_type+"-"+product.properties['id']).encode("utf-8")).hexdigest()``)
|
|
293
274
|
|
|
294
275
|
:param product: The product to calculate the record hash
|
|
295
|
-
:type product: :class:`~eodag.api.product._product.EOProduct`
|
|
296
276
|
:returns: The MD5 hash
|
|
297
|
-
:rtype: str
|
|
298
277
|
"""
|
|
299
278
|
# In some unit tests, `product.product_type` is `None` and `product.properties["id"]` is `ìnt`
|
|
300
279
|
product_hash = str(product.product_type) + "-" + str(product.properties["id"])
|
|
@@ -309,9 +288,7 @@ class Download(PluginTopic):
|
|
|
309
288
|
WARNING: A strong assumption is made here: there is only one subdirectory per level
|
|
310
289
|
|
|
311
290
|
:param product_path: The path to the extracted product
|
|
312
|
-
:type product_path: str
|
|
313
291
|
:returns: The path to the extracted product with the right depth
|
|
314
|
-
:rtype: str
|
|
315
292
|
"""
|
|
316
293
|
archive_depth = getattr(self.config, "archive_depth", 1)
|
|
317
294
|
count = 1
|
|
@@ -329,11 +306,8 @@ class Download(PluginTopic):
|
|
|
329
306
|
"""Finalize the download process.
|
|
330
307
|
|
|
331
308
|
:param fs_path: The path to the local zip archive downloaded or already present
|
|
332
|
-
:type fs_path: str
|
|
333
309
|
:param progress_callback: (optional) A progress callback
|
|
334
|
-
:type progress_callback: :class:`~eodag.utils.ProgressCallback` or None
|
|
335
310
|
:returns: The absolute path to the product
|
|
336
|
-
:rtype: str
|
|
337
311
|
"""
|
|
338
312
|
# progress bar init
|
|
339
313
|
if progress_callback is None:
|
|
@@ -365,11 +339,11 @@ class Download(PluginTopic):
|
|
|
365
339
|
if delete_archive is not None
|
|
366
340
|
else getattr(self.config, "delete_archive", True)
|
|
367
341
|
)
|
|
368
|
-
|
|
342
|
+
output_extension = kwargs.pop("output_extension", ".zip")
|
|
369
343
|
|
|
370
344
|
product_path = (
|
|
371
|
-
fs_path[: fs_path.index(
|
|
372
|
-
if
|
|
345
|
+
fs_path[: fs_path.index(output_extension)]
|
|
346
|
+
if output_extension in fs_path
|
|
373
347
|
else fs_path
|
|
374
348
|
)
|
|
375
349
|
product_path_exists = os.path.exists(product_path)
|
|
@@ -396,9 +370,7 @@ class Download(PluginTopic):
|
|
|
396
370
|
)
|
|
397
371
|
progress_callback(1, total=1)
|
|
398
372
|
return product_path
|
|
399
|
-
|
|
400
|
-
kwargs.pop("outputs_prefix", None) or self.config.outputs_prefix
|
|
401
|
-
)
|
|
373
|
+
output_dir = kwargs.pop("output_dir", None) or self.config.output_dir
|
|
402
374
|
|
|
403
375
|
if not os.path.exists(product_path):
|
|
404
376
|
logger.info("Extraction activated")
|
|
@@ -407,9 +379,9 @@ class Download(PluginTopic):
|
|
|
407
379
|
)
|
|
408
380
|
progress_callback.refresh()
|
|
409
381
|
|
|
410
|
-
|
|
382
|
+
product_dir = os.path.join(output_dir, product_path)
|
|
411
383
|
tmp_dir = tempfile.TemporaryDirectory()
|
|
412
|
-
extraction_dir = os.path.join(tmp_dir.name, os.path.basename(
|
|
384
|
+
extraction_dir = os.path.join(tmp_dir.name, os.path.basename(product_dir))
|
|
413
385
|
|
|
414
386
|
if fs_path.endswith(".zip"):
|
|
415
387
|
with zipfile.ZipFile(fs_path, "r") as zfile:
|
|
@@ -427,10 +399,10 @@ class Download(PluginTopic):
|
|
|
427
399
|
# then, we create a directory in which we place this file
|
|
428
400
|
product_extraction_path = self._resolve_archive_depth(extraction_dir)
|
|
429
401
|
if os.path.isfile(product_extraction_path) and not os.path.isdir(
|
|
430
|
-
|
|
402
|
+
product_dir
|
|
431
403
|
):
|
|
432
|
-
os.makedirs(
|
|
433
|
-
shutil.move(product_extraction_path,
|
|
404
|
+
os.makedirs(product_dir)
|
|
405
|
+
shutil.move(product_extraction_path, product_dir)
|
|
434
406
|
|
|
435
407
|
elif fs_path.endswith(".tar") or fs_path.endswith(".tar.gz"):
|
|
436
408
|
with tarfile.open(fs_path, "r") as zfile:
|
|
@@ -441,10 +413,10 @@ class Download(PluginTopic):
|
|
|
441
413
|
# then, we create a directory in which we place this file
|
|
442
414
|
product_extraction_path = self._resolve_archive_depth(extraction_dir)
|
|
443
415
|
if os.path.isfile(product_extraction_path) and not os.path.isdir(
|
|
444
|
-
|
|
416
|
+
product_dir
|
|
445
417
|
):
|
|
446
|
-
os.makedirs(
|
|
447
|
-
shutil.move(product_extraction_path,
|
|
418
|
+
os.makedirs(product_dir)
|
|
419
|
+
shutil.move(product_extraction_path, product_dir)
|
|
448
420
|
else:
|
|
449
421
|
progress_callback(1, total=1)
|
|
450
422
|
|
|
@@ -483,32 +455,23 @@ class Download(PluginTopic):
|
|
|
483
455
|
implemented by the plugin to **sequentially** attempt to download products.
|
|
484
456
|
|
|
485
457
|
:param products: Products to download
|
|
486
|
-
:type products: :class:`~eodag.api.search_result.SearchResult`
|
|
487
458
|
:param auth: (optional) authenticated object
|
|
488
|
-
:type auth: Optional[Union[AuthBase, Dict[str, str]]]
|
|
489
459
|
:param downloaded_callback: (optional) A method or a callable object which takes
|
|
490
460
|
as parameter the ``product``. You can use the base class
|
|
491
461
|
:class:`~eodag.api.product.DownloadedCallback` and override
|
|
492
462
|
its ``__call__`` method. Will be called each time a product
|
|
493
463
|
finishes downloading
|
|
494
|
-
:type downloaded_callback: Callable[[:class:`~eodag.api.product._product.EOProduct`], None]
|
|
495
|
-
or None
|
|
496
464
|
:param progress_callback: (optional) A progress callback
|
|
497
|
-
:type progress_callback: :class:`~eodag.utils.ProgressCallback`
|
|
498
465
|
:param wait: (optional) If download fails, wait time in minutes between two download tries
|
|
499
|
-
:type wait: int
|
|
500
466
|
:param timeout: (optional) If download fails, maximum time in minutes before stop retrying
|
|
501
467
|
to download
|
|
502
|
-
:
|
|
503
|
-
:param kwargs: `outputs_prefix` (str), `extract` (bool), `delete_archive` (bool)
|
|
468
|
+
:param kwargs: `output_dir` (str), `extract` (bool), `delete_archive` (bool)
|
|
504
469
|
and `dl_url_params` (dict) can be provided as additional kwargs
|
|
505
470
|
and will override any other values defined in a configuration
|
|
506
471
|
file or with environment variables.
|
|
507
|
-
:type kwargs: Union[str, bool, dict]
|
|
508
472
|
:returns: List of absolute paths to the downloaded products in the local
|
|
509
473
|
filesystem (e.g. ``['/tmp/product.zip']`` on Linux or
|
|
510
474
|
``['C:\\Users\\username\\AppData\\Local\\Temp\\product.zip']`` on Windows)
|
|
511
|
-
:rtype: list
|
|
512
475
|
"""
|
|
513
476
|
# Products are going to be removed one by one from this sequence once
|
|
514
477
|
# downloaded.
|
|
@@ -632,14 +595,10 @@ class Download(PluginTopic):
|
|
|
632
595
|
exception is thrown until `timeout` minutes.
|
|
633
596
|
|
|
634
597
|
:param product: The EO product to download
|
|
635
|
-
:type product: :class:`~eodag.api.product._product.EOProduct`
|
|
636
598
|
:param wait: If download fails, wait time in minutes between two download tries
|
|
637
|
-
:type wait: int
|
|
638
599
|
:param timeout: If download fails, maximum time in minutes before stop retrying
|
|
639
600
|
to download
|
|
640
|
-
:type timeout: int
|
|
641
601
|
:returns: decorator
|
|
642
|
-
:rtype: Callable[[Callable[..., T]], Callable[..., T]]
|
|
643
602
|
"""
|
|
644
603
|
|
|
645
604
|
def decorator(download: Callable[..., T]) -> Callable[..., T]:
|
|
@@ -670,7 +629,7 @@ class Download(PluginTopic):
|
|
|
670
629
|
not_available_info = str(e)
|
|
671
630
|
|
|
672
631
|
if datetime_now >= product.next_try and datetime_now < stop_time:
|
|
673
|
-
wait_seconds = (
|
|
632
|
+
wait_seconds: Union[float, int] = (
|
|
674
633
|
datetime_now - product.next_try + timedelta(minutes=wait)
|
|
675
634
|
).seconds
|
|
676
635
|
retry_count += 1
|