eodag 3.0.0b1__py3-none-any.whl → 3.0.0b3__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (66) hide show
  1. eodag/__init__.py +6 -8
  2. eodag/api/core.py +119 -171
  3. eodag/api/product/__init__.py +10 -4
  4. eodag/api/product/_assets.py +52 -14
  5. eodag/api/product/_product.py +59 -30
  6. eodag/api/product/drivers/__init__.py +7 -2
  7. eodag/api/product/drivers/base.py +0 -3
  8. eodag/api/product/metadata_mapping.py +0 -28
  9. eodag/api/search_result.py +31 -9
  10. eodag/config.py +45 -41
  11. eodag/plugins/apis/base.py +3 -3
  12. eodag/plugins/apis/ecmwf.py +2 -3
  13. eodag/plugins/apis/usgs.py +43 -14
  14. eodag/plugins/authentication/aws_auth.py +11 -2
  15. eodag/plugins/authentication/openid_connect.py +5 -4
  16. eodag/plugins/authentication/token.py +2 -1
  17. eodag/plugins/crunch/base.py +3 -1
  18. eodag/plugins/crunch/filter_date.py +3 -9
  19. eodag/plugins/crunch/filter_latest_intersect.py +0 -3
  20. eodag/plugins/crunch/filter_latest_tpl_name.py +1 -4
  21. eodag/plugins/crunch/filter_overlap.py +4 -8
  22. eodag/plugins/crunch/filter_property.py +5 -11
  23. eodag/plugins/download/aws.py +46 -78
  24. eodag/plugins/download/base.py +27 -68
  25. eodag/plugins/download/http.py +48 -57
  26. eodag/plugins/download/s3rest.py +17 -25
  27. eodag/plugins/manager.py +6 -18
  28. eodag/plugins/search/__init__.py +9 -9
  29. eodag/plugins/search/base.py +7 -26
  30. eodag/plugins/search/build_search_result.py +0 -13
  31. eodag/plugins/search/cop_marine.py +1 -3
  32. eodag/plugins/search/creodias_s3.py +0 -3
  33. eodag/plugins/search/data_request_search.py +10 -5
  34. eodag/plugins/search/qssearch.py +95 -53
  35. eodag/plugins/search/static_stac_search.py +6 -3
  36. eodag/resources/ext_product_types.json +1 -1
  37. eodag/resources/product_types.yml +24 -0
  38. eodag/resources/providers.yml +198 -154
  39. eodag/resources/user_conf_template.yml +27 -27
  40. eodag/rest/core.py +11 -43
  41. eodag/rest/server.py +1 -6
  42. eodag/rest/stac.py +13 -87
  43. eodag/rest/types/eodag_search.py +4 -7
  44. eodag/rest/types/queryables.py +4 -12
  45. eodag/rest/types/stac_search.py +7 -11
  46. eodag/rest/utils/rfc3339.py +0 -1
  47. eodag/types/__init__.py +9 -3
  48. eodag/types/download_args.py +14 -5
  49. eodag/types/search_args.py +7 -8
  50. eodag/types/whoosh.py +0 -2
  51. eodag/utils/__init__.py +20 -79
  52. eodag/utils/constraints.py +0 -8
  53. eodag/utils/import_system.py +0 -4
  54. eodag/utils/logging.py +0 -3
  55. eodag/utils/notebook.py +4 -4
  56. eodag/utils/repr.py +113 -0
  57. eodag/utils/requests.py +12 -20
  58. eodag/utils/rest.py +0 -4
  59. eodag/utils/stac_reader.py +2 -14
  60. {eodag-3.0.0b1.dist-info → eodag-3.0.0b3.dist-info}/METADATA +33 -14
  61. eodag-3.0.0b3.dist-info/RECORD +110 -0
  62. {eodag-3.0.0b1.dist-info → eodag-3.0.0b3.dist-info}/WHEEL +1 -1
  63. eodag-3.0.0b1.dist-info/RECORD +0 -109
  64. {eodag-3.0.0b1.dist-info → eodag-3.0.0b3.dist-info}/LICENSE +0 -0
  65. {eodag-3.0.0b1.dist-info → eodag-3.0.0b3.dist-info}/entry_points.txt +0 -0
  66. {eodag-3.0.0b1.dist-info → eodag-3.0.0b3.dist-info}/top_level.txt +0 -0
@@ -19,7 +19,7 @@ from __future__ import annotations
19
19
 
20
20
  import logging
21
21
  import operator
22
- from typing import TYPE_CHECKING, Any, Dict, List, Optional, Union
22
+ from typing import TYPE_CHECKING, Any, List
23
23
 
24
24
  from eodag.plugins.crunch.base import Crunch
25
25
 
@@ -38,23 +38,17 @@ class FilterProperty(Crunch):
38
38
 
39
39
  - `property=value` : property key from product.properties, associated to its filter value
40
40
  - `operator` : (optional) Operator used for filtering (one of `lt,le,eq,ne,ge,gt`). Default is `eq`
41
-
42
- :type config: dict
43
41
  """
44
42
 
45
- config: Dict[str, Union[str, Optional[str]]]
46
-
47
43
  def proceed(
48
44
  self, products: List[EOProduct], **search_params: Any
49
45
  ) -> List[EOProduct]:
50
46
  """Execute crunch: Filter products, retaining only those that match property filtering
51
47
 
52
48
  :param products: A list of products resulting from a search
53
- :type products: list(:class:`~eodag.api.product._product.EOProduct`)
54
49
  :returns: The filtered products
55
- :rtype: list(:class:`~eodag.api.product._product.EOProduct`)
56
50
  """
57
- operator_name = self.config.pop("operator", "eq") or "eq"
51
+ operator_name = self.config.__dict__.pop("operator", "eq") or "eq"
58
52
  try:
59
53
  operator_method = getattr(operator, operator_name)
60
54
  except AttributeError:
@@ -64,12 +58,12 @@ class FilterProperty(Crunch):
64
58
  )
65
59
  return products
66
60
 
67
- if len(self.config.keys()) != 1:
61
+ if len(self.config.__dict__.keys()) != 1:
68
62
  logger.warning("One property is needed for filtering, filtering disabled.")
69
63
  return products
70
64
 
71
- property_key = next(iter(self.config))
72
- property_value = self.config.get(property_key, None)
65
+ property_key = next(iter(self.config.__dict__))
66
+ property_value = self.config.__dict__.get(property_key, None)
73
67
 
74
68
  logger.debug(
75
69
  "Start filtering for products matching operator.%s(product.properties['%s'], %s)",
@@ -34,6 +34,7 @@ from typing import (
34
34
  Optional,
35
35
  Set,
36
36
  Tuple,
37
+ TypedDict,
37
38
  Union,
38
39
  cast,
39
40
  )
@@ -213,7 +214,6 @@ class AwsDownload(Download):
213
214
  """Download on AWS using S3 protocol.
214
215
 
215
216
  :param provider: provider name
216
- :type provider: str
217
217
  :param config: Download plugin configuration:
218
218
 
219
219
  * ``config.base_uri`` (str) - s3 endpoint url
@@ -222,14 +222,12 @@ class AwsDownload(Download):
222
222
  * ``config.flatten_top_dirs`` (bool) - (optional) flatten directory structure
223
223
  * ``config.products`` (dict) - (optional) product_type specific configuration
224
224
  * ``config.ignore_assets`` (bool) - (optional) ignore assets and download using downloadLink
225
-
226
- :type config: :class:`~eodag.config.PluginConfig`
227
225
  """
228
226
 
229
227
  def __init__(self, provider: str, config: PluginConfig) -> None:
230
228
  super(AwsDownload, self).__init__(provider, config)
231
229
  self.requester_pays = getattr(self.config, "requester_pays", False)
232
- self.s3_session = None
230
+ self.s3_session: Optional[boto3.session.Session] = None
233
231
 
234
232
  def download(
235
233
  self,
@@ -246,25 +244,20 @@ class AwsDownload(Download):
246
244
  SAFE-build is configured for a given provider and product type.
247
245
  If the product title is configured to be updated during download and
248
246
  SAFE-formatted, its destination path will be:
249
- `{outputs_prefix}/{title}`
247
+ `{output_dir}/{title}`
250
248
 
251
249
  :param product: The EO product to download
252
- :type product: :class:`~eodag.api.product._product.EOProduct`
253
250
  :param auth: (optional) authenticated object
254
- :type auth: Union[AuthBase, Dict[str, str]]
255
251
  :param progress_callback: (optional) A method or a callable object
256
252
  which takes a current size and a maximum
257
253
  size as inputs and handle progress bar
258
254
  creation and update to give the user a
259
255
  feedback on the download progress
260
- :type progress_callback: :class:`~eodag.utils.ProgressCallback` or None
261
- :param kwargs: `outputs_prefix` (str), `extract` (bool), `delete_archive` (bool)
256
+ :param kwargs: `output_dir` (str), `extract` (bool), `delete_archive` (bool)
262
257
  and `dl_url_params` (dict) can be provided as additional kwargs
263
258
  and will override any other values defined in a configuration
264
259
  file or with environment variables.
265
- :type kwargs: Union[str, bool, dict]
266
260
  :returns: The absolute path to the downloaded product in the local filesystem
267
- :rtype: str
268
261
  """
269
262
  if auth is None:
270
263
  auth = {}
@@ -404,13 +397,9 @@ class AwsDownload(Download):
404
397
  - get file path
405
398
  - create directories
406
399
  :param product: product to be downloaded
407
- :type product: EOProduct
408
400
  :param progress_callback: progress callback to be used
409
- :type progress_callback: ProgressCallback
410
401
  :param kwargs: additional arguments
411
- :type kwargs: Any
412
402
  :return: local path and file name
413
- :rtype: Tuple[str, Optional[str]]
414
403
  """
415
404
  product_local_path, record_filename = self._prepare_download(
416
405
  product, progress_callback=progress_callback, **kwargs
@@ -432,9 +421,7 @@ class AwsDownload(Download):
432
421
  """
433
422
  updates the product properties with fetch metadata if safe build is enabled
434
423
  :param build_safe: if safe build is enabled
435
- :type build_safe: bool
436
424
  :param product: product to be updated
437
- :type product: EOProduct
438
425
  """
439
426
  product_conf = getattr(self.config, "products", {}).get(
440
427
  product.product_type, {}
@@ -480,13 +467,9 @@ class AwsDownload(Download):
480
467
  """
481
468
  retrieves the bucket names and path prefixes for the assets
482
469
  :param product: product for which the assets shall be downloaded
483
- :type product: EOProduct
484
470
  :param asset_filter: text for which the assets should be filtered
485
- :type asset_filter: str
486
471
  :param ignore_assets: if product instead of individual assets should be used
487
- :type ignore_assets: bool
488
472
  :return: tuples of bucket names and prefixes
489
- :rtype: List[Tuple[str, Optional[str]]]
490
473
  """
491
474
  # if assets are defined, use them instead of scanning product.location
492
475
  if len(product.assets) > 0 and not ignore_assets:
@@ -528,11 +511,8 @@ class AwsDownload(Download):
528
511
  authenticates with s3 and retrieves the available objects
529
512
  raises an error when authentication is not possible
530
513
  :param bucket_names_and_prefixes: list of bucket names and corresponding path prefixes
531
- :type bucket_names_and_prefixes: List[Tuple[str, Optional[str]]]
532
514
  :param auth: authentication information
533
- :type auth: Dict[str, str]
534
515
  :return: authenticated objects per bucket, list of available objects
535
- :rtype: Tuple[Dict[str, Any], ResourceCollection[Any]]
536
516
  """
537
517
  if not isinstance(auth, (dict, type(None))):
538
518
  raise AuthenticationError(
@@ -602,17 +582,11 @@ class AwsDownload(Download):
602
582
  """
603
583
  retrieve unique product chunks based on authenticated objects and asset filters
604
584
  :param bucket_names_and_prefixes: list of bucket names and corresponding path prefixes
605
- :type bucket_names_and_prefixes: List[Tuple[str, Optional[str]]]
606
585
  :param authenticated_objects: available objects per bucket
607
- :type authenticated_objects: Dict[str, Any]
608
586
  :param asset_filter: text for which assets should be filtered
609
- :type asset_filter: str
610
587
  :param ignore_assets: if product instead of individual assets should be used
611
- :type ignore_assets: bool
612
588
  :param product: product that shall be downloaded
613
- :type product: EOProduct
614
589
  :return: set of product chunks that can be downloaded
615
- :rtype: Set[Any]
616
590
  """
617
591
  product_chunks: List[Any] = []
618
592
  for bucket_name, prefix in bucket_names_and_prefixes:
@@ -666,23 +640,16 @@ class AwsDownload(Download):
666
640
  It contains a generator to streamed download chunks and the response headers.
667
641
 
668
642
  :param product: The EO product to download
669
- :type product: :class:`~eodag.api.product._product.EOProduct`
670
643
  :param auth: (optional) The configuration of a plugin of type Authentication
671
- :type auth: :class:`~eodag.config.PluginConfig`
672
644
  :param progress_callback: (optional) A progress callback
673
- :type progress_callback: :class:`~eodag.utils.ProgressCallback`
674
645
  :param wait: (optional) If download fails, wait time in minutes between two download tries
675
- :type wait: int
676
646
  :param timeout: (optional) If download fails, maximum time in minutes before stop retrying
677
647
  to download
678
- :type timeout: int
679
- :param kwargs: `outputs_prefix` (str), `extract` (bool), `delete_archive` (bool)
648
+ :param kwargs: `output_dir` (str), `extract` (bool), `delete_archive` (bool)
680
649
  and `dl_url_params` (dict) can be provided as additional kwargs
681
650
  and will override any other values defined in a configuration
682
651
  file or with environment variables.
683
- :type kwargs: Union[str, bool, dict]
684
652
  :returns: Dictionnary of :class:`~fastapi.responses.StreamingResponse` keyword-arguments
685
- :rtype: dict
686
653
  """
687
654
  if progress_callback is None:
688
655
  logger.info(
@@ -773,7 +740,7 @@ class AwsDownload(Download):
773
740
  build_safe: bool,
774
741
  progress_callback: ProgressCallback,
775
742
  assets_values: List[Dict[str, Any]],
776
- ) -> Iterator[Tuple[str, datetime, int, Any, Iterator[Any]]]:
743
+ ) -> Iterator[Any]:
777
744
  """Yield product data chunks"""
778
745
 
779
746
  chunk_size = 4096 * 1024
@@ -861,13 +828,9 @@ class AwsDownload(Download):
861
828
  """Get rasterio environment variables needed for data access authentication.
862
829
 
863
830
  :param bucket_name: Bucket containg objects
864
- :type bucket_name: str
865
831
  :param prefix: Prefix used to try auth
866
- :type prefix: str
867
832
  :param auth_dict: Dictionnary containing authentication keys
868
- :type auth_dict: dict
869
833
  :returns: The rasterio environement variables
870
- :rtype: dict
871
834
  """
872
835
  if self.s3_session is not None:
873
836
  if self.requester_pays:
@@ -892,14 +855,10 @@ class AwsDownload(Download):
892
855
  Also expose ``s3_session`` as class variable if available.
893
856
 
894
857
  :param bucket_name: Bucket containg objects
895
- :type bucket_name: str
896
858
  :param prefix: Prefix used to filter objects on auth try
897
859
  (not used to filter returned objects)
898
- :type prefix: str
899
860
  :param auth_dict: Dictionnary containing authentication keys
900
- :type auth_dict: dict
901
861
  :returns: The boto3 authenticated objects
902
- :rtype: :class:`~boto3.resources.collection.s3.Bucket.objectsCollection`
903
862
  """
904
863
  auth_methods: List[
905
864
  Callable[[str, str, Dict[str, str]], Optional[ResourceCollection]]
@@ -941,15 +900,15 @@ class AwsDownload(Download):
941
900
  ) -> Optional[ResourceCollection]:
942
901
  """Auth strategy using no-sign-request"""
943
902
 
944
- s3_resource = boto3.resource( # type: ignore
903
+ s3_resource = boto3.resource(
945
904
  service_name="s3", endpoint_url=getattr(self.config, "base_uri", None)
946
905
  )
947
- s3_resource.meta.client.meta.events.register( # type: ignore
906
+ s3_resource.meta.client.meta.events.register(
948
907
  "choose-signer.s3.*", disable_signing
949
908
  )
950
- objects = s3_resource.Bucket(bucket_name).objects # type: ignore
951
- list(objects.filter(Prefix=prefix).limit(1)) # type: ignore
952
- return objects # type: ignore
909
+ objects = s3_resource.Bucket(bucket_name).objects
910
+ list(objects.filter(Prefix=prefix).limit(1))
911
+ return objects
953
912
 
954
913
  def _get_authenticated_objects_from_auth_profile(
955
914
  self, bucket_name: str, prefix: str, auth_dict: Dict[str, str]
@@ -957,20 +916,20 @@ class AwsDownload(Download):
957
916
  """Auth strategy using RequestPayer=requester and ``aws_profile`` from provided credentials"""
958
917
 
959
918
  if "profile_name" in auth_dict.keys():
960
- s3_session = boto3.session.Session(profile_name=auth_dict["profile_name"]) # type: ignore
961
- s3_resource = s3_session.resource( # type: ignore
919
+ s3_session = boto3.session.Session(profile_name=auth_dict["profile_name"])
920
+ s3_resource = s3_session.resource(
962
921
  service_name="s3",
963
922
  endpoint_url=getattr(self.config, "base_uri", None),
964
923
  )
965
924
  if self.requester_pays:
966
- objects = s3_resource.Bucket(bucket_name).objects.filter( # type: ignore
925
+ objects = s3_resource.Bucket(bucket_name).objects.filter(
967
926
  RequestPayer="requester"
968
927
  )
969
928
  else:
970
- objects = s3_resource.Bucket(bucket_name).objects # type: ignore
971
- list(objects.filter(Prefix=prefix).limit(1)) # type: ignore
972
- self.s3_session = s3_session # type: ignore
973
- return objects # type: ignore
929
+ objects = s3_resource.Bucket(bucket_name).objects
930
+ list(objects.filter(Prefix=prefix).limit(1))
931
+ self.s3_session = s3_session
932
+ return objects
974
933
  else:
975
934
  return None
976
935
 
@@ -981,23 +940,35 @@ class AwsDownload(Download):
981
940
  from provided credentials"""
982
941
 
983
942
  if all(k in auth_dict for k in ("aws_access_key_id", "aws_secret_access_key")):
984
- s3_session = boto3.session.Session( # type: ignore
985
- aws_access_key_id=auth_dict["aws_access_key_id"],
986
- aws_secret_access_key=auth_dict["aws_secret_access_key"],
943
+ S3SessionKwargs = TypedDict(
944
+ "S3SessionKwargs",
945
+ {
946
+ "aws_access_key_id": str,
947
+ "aws_secret_access_key": str,
948
+ "aws_session_token": str,
949
+ },
950
+ total=False,
987
951
  )
988
- s3_resource = s3_session.resource( # type: ignore
952
+ s3_session_kwargs: S3SessionKwargs = {
953
+ "aws_access_key_id": auth_dict["aws_access_key_id"],
954
+ "aws_secret_access_key": auth_dict["aws_secret_access_key"],
955
+ }
956
+ if auth_dict.get("aws_session_token"):
957
+ s3_session_kwargs["aws_session_token"] = auth_dict["aws_session_token"]
958
+ s3_session = boto3.session.Session(**s3_session_kwargs)
959
+ s3_resource = s3_session.resource(
989
960
  service_name="s3",
990
961
  endpoint_url=getattr(self.config, "base_uri", None),
991
962
  )
992
963
  if self.requester_pays:
993
- objects = s3_resource.Bucket(bucket_name).objects.filter( # type: ignore
964
+ objects = s3_resource.Bucket(bucket_name).objects.filter(
994
965
  RequestPayer="requester"
995
966
  )
996
967
  else:
997
- objects = s3_resource.Bucket(bucket_name).objects # type: ignore
998
- list(objects.filter(Prefix=prefix).limit(1)) # type: ignore
999
- self.s3_session = s3_session # type: ignore
1000
- return objects # type: ignore
968
+ objects = s3_resource.Bucket(bucket_name).objects
969
+ list(objects.filter(Prefix=prefix).limit(1))
970
+ self.s3_session = s3_session
971
+ return objects
1001
972
  else:
1002
973
  return None
1003
974
 
@@ -1006,19 +977,19 @@ class AwsDownload(Download):
1006
977
  ) -> Optional[ResourceCollection]:
1007
978
  """Auth strategy using RequestPayer=requester and current environment"""
1008
979
 
1009
- s3_session = boto3.session.Session() # type: ignore
1010
- s3_resource = s3_session.resource( # type: ignore
980
+ s3_session = boto3.session.Session()
981
+ s3_resource = s3_session.resource(
1011
982
  service_name="s3", endpoint_url=getattr(self.config, "base_uri", None)
1012
983
  )
1013
984
  if self.requester_pays:
1014
- objects = s3_resource.Bucket(bucket_name).objects.filter( # type: ignore
985
+ objects = s3_resource.Bucket(bucket_name).objects.filter(
1015
986
  RequestPayer="requester"
1016
987
  )
1017
988
  else:
1018
- objects = s3_resource.Bucket(bucket_name).objects # type: ignore
1019
- list(objects.filter(Prefix=prefix).limit(1)) # type: ignore
1020
- self.s3_session = s3_session # type: ignore
1021
- return objects # type: ignore
989
+ objects = s3_resource.Bucket(bucket_name).objects
990
+ list(objects.filter(Prefix=prefix).limit(1))
991
+ self.s3_session = s3_session
992
+ return objects
1022
993
 
1023
994
  def get_product_bucket_name_and_prefix(
1024
995
  self, product: EOProduct, url: Optional[str] = None
@@ -1026,11 +997,8 @@ class AwsDownload(Download):
1026
997
  """Extract bucket name and prefix from product URL
1027
998
 
1028
999
  :param product: The EO product to download
1029
- :type product: :class:`~eodag.api.product._product.EOProduct`
1030
1000
  :param url: (optional) URL to use as product.location
1031
- :type url: str
1032
1001
  :returns: bucket_name and prefix as str
1033
- :rtype: tuple
1034
1002
  """
1035
1003
  if url is None:
1036
1004
  url = product.location
@@ -79,17 +79,17 @@ class Download(PluginTopic):
79
79
 
80
80
  They must:
81
81
 
82
- - download data in the ``outputs_prefix`` folder defined in the plugin's
82
+ - download data in the ``output_dir`` folder defined in the plugin's
83
83
  configuration or passed through kwargs
84
84
  - extract products from their archive (if relevant) if ``extract`` is set to True
85
85
  (True by default)
86
- - save a product in an archive/directory (in ``outputs_prefix``) whose name must be
86
+ - save a product in an archive/directory (in ``output_dir``) whose name must be
87
87
  the product's ``title`` property
88
88
  - update the product's ``location`` attribute once its data is downloaded (and
89
89
  eventually after it's extracted) to the product's location given as a file URI
90
90
  (e.g. 'file:///tmp/product_folder' on Linux or
91
91
  'file:///C:/Users/username/AppData/LOcal/Temp' on Windows)
92
- - save a *record* file in the directory ``outputs_prefix/.downloaded`` whose name
92
+ - save a *record* file in the directory ``output_dir/.downloaded`` whose name
93
93
  is built on the MD5 hash of the product's ``product_type`` and ``properties['id']``
94
94
  attributes (``hashlib.md5((product.product_type+"-"+product.properties['id']).encode("utf-8")).hexdigest()``)
95
95
  and whose content is the product's ``remote_location`` attribute itself.
@@ -100,9 +100,7 @@ class Download(PluginTopic):
100
100
  (it certainly indicates that the download didn't complete)
101
101
 
102
102
  :param provider: An eodag providers configuration dictionary
103
- :type provider: dict
104
103
  :param config: Path to the user configuration file
105
- :type config: str
106
104
  """
107
105
 
108
106
  def __init__(self, provider: str, config: PluginConfig) -> None:
@@ -122,25 +120,18 @@ class Download(PluginTopic):
122
120
  Base download method. Not available, it must be defined for each plugin.
123
121
 
124
122
  :param product: The EO product to download
125
- :type product: :class:`~eodag.api.product._product.EOProduct`
126
123
  :param auth: (optional) authenticated object
127
- :type auth: Optional[Union[AuthBase, Dict[str, str]]]
128
124
  :param progress_callback: (optional) A progress callback
129
- :type progress_callback: :class:`~eodag.utils.ProgressCallback`
130
125
  :param wait: (optional) If download fails, wait time in minutes between two download tries
131
- :type wait: int
132
126
  :param timeout: (optional) If download fails, maximum time in minutes before stop retrying
133
127
  to download
134
- :type timeout: int
135
- :param kwargs: `outputs_prefix` (str), `extract` (bool), `delete_archive` (bool)
128
+ :param kwargs: `output_dir` (str), `extract` (bool), `delete_archive` (bool)
136
129
  and `dl_url_params` (dict) can be provided as additional kwargs
137
130
  and will override any other values defined in a configuration
138
131
  file or with environment variables.
139
- :type kwargs: Union[str, bool, dict]
140
132
  :returns: The absolute path to the downloaded product in the local filesystem
141
133
  (e.g. '/tmp/product.zip' on Linux or
142
134
  'C:\\Users\\username\\AppData\\Local\\Temp\\product.zip' on Windows)
143
- :rtype: str
144
135
  """
145
136
  raise NotImplementedError(
146
137
  "A Download plugin must implement a method named download"
@@ -159,23 +150,16 @@ class Download(PluginTopic):
159
150
  Base _stream_download_dict method. Not available, it must be defined for each plugin.
160
151
 
161
152
  :param product: The EO product to download
162
- :type product: :class:`~eodag.api.product._product.EOProduct`
163
153
  :param auth: (optional) authenticated object
164
- :type auth: Optional[Union[AuthBase, Dict[str, str]]]
165
154
  :param progress_callback: (optional) A progress callback
166
- :type progress_callback: :class:`~eodag.utils.ProgressCallback`
167
155
  :param wait: (optional) If download fails, wait time in minutes between two download tries
168
- :type wait: int
169
156
  :param timeout: (optional) If download fails, maximum time in minutes before stop retrying
170
157
  to download
171
- :type timeout: int
172
- :param kwargs: `outputs_prefix` (str), `extract` (bool), `delete_archive` (bool)
158
+ :param kwargs: `output_dir` (str), `extract` (bool), `delete_archive` (bool)
173
159
  and `dl_url_params` (dict) can be provided as additional kwargs
174
160
  and will override any other values defined in a configuration
175
161
  file or with environment variables.
176
- :type kwargs: Union[str, bool, dict]
177
162
  :returns: Dictionnary of :class:`~fastapi.responses.StreamingResponse` keyword-arguments
178
- :rtype: dict
179
163
  """
180
164
  raise NotImplementedError(
181
165
  "Download streaming must be implemented using a method named _stream_download_dict"
@@ -190,11 +174,8 @@ class Download(PluginTopic):
190
174
  """Check if file has already been downloaded, and prepare product download
191
175
 
192
176
  :param product: The EO product to download
193
- :type product: :class:`~eodag.api.product._product.EOProduct`
194
177
  :param progress_callback: (optional) A progress callback
195
- :type progress_callback: :class:`~eodag.utils.ProgressCallback` or None
196
178
  :returns: fs_path, record_filename
197
- :rtype: Tuple[Optional[str], Optional[str]]
198
179
  """
199
180
  if product.location != product.remote_location:
200
181
  fs_path = uri_to_path(product.location)
@@ -216,18 +197,18 @@ class Download(PluginTopic):
216
197
  f"Download url: {url}",
217
198
  )
218
199
 
219
- outputs_prefix = (
220
- kwargs.pop("outputs_prefix", None)
221
- or getattr(self.config, "outputs_prefix", tempfile.gettempdir())
200
+ output_dir = (
201
+ kwargs.pop("output_dir", None)
202
+ or getattr(self.config, "output_dir", tempfile.gettempdir())
222
203
  or tempfile.gettempdir()
223
204
  )
224
- outputs_extension = kwargs.get("outputs_extension", None) or getattr(
225
- self.config, "outputs_extension", ".zip"
205
+ output_extension = kwargs.get("output_extension", None) or getattr(
206
+ self.config, "output_extension", ".zip"
226
207
  )
227
208
 
228
209
  # Strong asumption made here: all products downloaded will be zip files
229
210
  # If they are not, the '.zip' extension will be removed when they are downloaded and returned as is
230
- prefix = os.path.abspath(outputs_prefix)
211
+ prefix = os.path.abspath(output_dir)
231
212
  sanitized_title = sanitize(product.properties["title"])
232
213
  if sanitized_title == product.properties["title"]:
233
214
  collision_avoidance_suffix = ""
@@ -235,10 +216,10 @@ class Download(PluginTopic):
235
216
  collision_avoidance_suffix = "-" + sanitize(product.properties["id"])
236
217
  fs_path = os.path.join(
237
218
  prefix,
238
- f"{sanitize(product.properties['title'])}{collision_avoidance_suffix}{outputs_extension}",
219
+ f"{sanitize(product.properties['title'])}{collision_avoidance_suffix}{output_extension}",
239
220
  )
240
221
  fs_dir_path = (
241
- fs_path.replace(outputs_extension, "") if outputs_extension else fs_path
222
+ fs_path.replace(output_extension, "") if output_extension else fs_path
242
223
  )
243
224
  download_records_dir = os.path.join(prefix, ".downloaded")
244
225
  try:
@@ -292,9 +273,7 @@ class Download(PluginTopic):
292
273
  (``hashlib.md5((product.product_type+"-"+product.properties['id']).encode("utf-8")).hexdigest()``)
293
274
 
294
275
  :param product: The product to calculate the record hash
295
- :type product: :class:`~eodag.api.product._product.EOProduct`
296
276
  :returns: The MD5 hash
297
- :rtype: str
298
277
  """
299
278
  # In some unit tests, `product.product_type` is `None` and `product.properties["id"]` is `ìnt`
300
279
  product_hash = str(product.product_type) + "-" + str(product.properties["id"])
@@ -309,9 +288,7 @@ class Download(PluginTopic):
309
288
  WARNING: A strong assumption is made here: there is only one subdirectory per level
310
289
 
311
290
  :param product_path: The path to the extracted product
312
- :type product_path: str
313
291
  :returns: The path to the extracted product with the right depth
314
- :rtype: str
315
292
  """
316
293
  archive_depth = getattr(self.config, "archive_depth", 1)
317
294
  count = 1
@@ -329,11 +306,8 @@ class Download(PluginTopic):
329
306
  """Finalize the download process.
330
307
 
331
308
  :param fs_path: The path to the local zip archive downloaded or already present
332
- :type fs_path: str
333
309
  :param progress_callback: (optional) A progress callback
334
- :type progress_callback: :class:`~eodag.utils.ProgressCallback` or None
335
310
  :returns: The absolute path to the product
336
- :rtype: str
337
311
  """
338
312
  # progress bar init
339
313
  if progress_callback is None:
@@ -365,11 +339,11 @@ class Download(PluginTopic):
365
339
  if delete_archive is not None
366
340
  else getattr(self.config, "delete_archive", True)
367
341
  )
368
- outputs_extension = kwargs.pop("outputs_extension", ".zip")
342
+ output_extension = kwargs.pop("output_extension", ".zip")
369
343
 
370
344
  product_path = (
371
- fs_path[: fs_path.index(outputs_extension)]
372
- if outputs_extension in fs_path
345
+ fs_path[: fs_path.index(output_extension)]
346
+ if output_extension in fs_path
373
347
  else fs_path
374
348
  )
375
349
  product_path_exists = os.path.exists(product_path)
@@ -396,9 +370,7 @@ class Download(PluginTopic):
396
370
  )
397
371
  progress_callback(1, total=1)
398
372
  return product_path
399
- outputs_prefix = (
400
- kwargs.pop("outputs_prefix", None) or self.config.outputs_prefix
401
- )
373
+ output_dir = kwargs.pop("output_dir", None) or self.config.output_dir
402
374
 
403
375
  if not os.path.exists(product_path):
404
376
  logger.info("Extraction activated")
@@ -407,9 +379,9 @@ class Download(PluginTopic):
407
379
  )
408
380
  progress_callback.refresh()
409
381
 
410
- outputs_dir = os.path.join(outputs_prefix, product_path)
382
+ product_dir = os.path.join(output_dir, product_path)
411
383
  tmp_dir = tempfile.TemporaryDirectory()
412
- extraction_dir = os.path.join(tmp_dir.name, os.path.basename(outputs_dir))
384
+ extraction_dir = os.path.join(tmp_dir.name, os.path.basename(product_dir))
413
385
 
414
386
  if fs_path.endswith(".zip"):
415
387
  with zipfile.ZipFile(fs_path, "r") as zfile:
@@ -427,10 +399,10 @@ class Download(PluginTopic):
427
399
  # then, we create a directory in which we place this file
428
400
  product_extraction_path = self._resolve_archive_depth(extraction_dir)
429
401
  if os.path.isfile(product_extraction_path) and not os.path.isdir(
430
- outputs_dir
402
+ product_dir
431
403
  ):
432
- os.makedirs(outputs_dir)
433
- shutil.move(product_extraction_path, outputs_dir)
404
+ os.makedirs(product_dir)
405
+ shutil.move(product_extraction_path, product_dir)
434
406
 
435
407
  elif fs_path.endswith(".tar") or fs_path.endswith(".tar.gz"):
436
408
  with tarfile.open(fs_path, "r") as zfile:
@@ -441,10 +413,10 @@ class Download(PluginTopic):
441
413
  # then, we create a directory in which we place this file
442
414
  product_extraction_path = self._resolve_archive_depth(extraction_dir)
443
415
  if os.path.isfile(product_extraction_path) and not os.path.isdir(
444
- outputs_dir
416
+ product_dir
445
417
  ):
446
- os.makedirs(outputs_dir)
447
- shutil.move(product_extraction_path, outputs_dir)
418
+ os.makedirs(product_dir)
419
+ shutil.move(product_extraction_path, product_dir)
448
420
  else:
449
421
  progress_callback(1, total=1)
450
422
 
@@ -483,32 +455,23 @@ class Download(PluginTopic):
483
455
  implemented by the plugin to **sequentially** attempt to download products.
484
456
 
485
457
  :param products: Products to download
486
- :type products: :class:`~eodag.api.search_result.SearchResult`
487
458
  :param auth: (optional) authenticated object
488
- :type auth: Optional[Union[AuthBase, Dict[str, str]]]
489
459
  :param downloaded_callback: (optional) A method or a callable object which takes
490
460
  as parameter the ``product``. You can use the base class
491
461
  :class:`~eodag.api.product.DownloadedCallback` and override
492
462
  its ``__call__`` method. Will be called each time a product
493
463
  finishes downloading
494
- :type downloaded_callback: Callable[[:class:`~eodag.api.product._product.EOProduct`], None]
495
- or None
496
464
  :param progress_callback: (optional) A progress callback
497
- :type progress_callback: :class:`~eodag.utils.ProgressCallback`
498
465
  :param wait: (optional) If download fails, wait time in minutes between two download tries
499
- :type wait: int
500
466
  :param timeout: (optional) If download fails, maximum time in minutes before stop retrying
501
467
  to download
502
- :type timeout: int
503
- :param kwargs: `outputs_prefix` (str), `extract` (bool), `delete_archive` (bool)
468
+ :param kwargs: `output_dir` (str), `extract` (bool), `delete_archive` (bool)
504
469
  and `dl_url_params` (dict) can be provided as additional kwargs
505
470
  and will override any other values defined in a configuration
506
471
  file or with environment variables.
507
- :type kwargs: Union[str, bool, dict]
508
472
  :returns: List of absolute paths to the downloaded products in the local
509
473
  filesystem (e.g. ``['/tmp/product.zip']`` on Linux or
510
474
  ``['C:\\Users\\username\\AppData\\Local\\Temp\\product.zip']`` on Windows)
511
- :rtype: list
512
475
  """
513
476
  # Products are going to be removed one by one from this sequence once
514
477
  # downloaded.
@@ -632,14 +595,10 @@ class Download(PluginTopic):
632
595
  exception is thrown until `timeout` minutes.
633
596
 
634
597
  :param product: The EO product to download
635
- :type product: :class:`~eodag.api.product._product.EOProduct`
636
598
  :param wait: If download fails, wait time in minutes between two download tries
637
- :type wait: int
638
599
  :param timeout: If download fails, maximum time in minutes before stop retrying
639
600
  to download
640
- :type timeout: int
641
601
  :returns: decorator
642
- :rtype: Callable[[Callable[..., T]], Callable[..., T]]
643
602
  """
644
603
 
645
604
  def decorator(download: Callable[..., T]) -> Callable[..., T]:
@@ -670,7 +629,7 @@ class Download(PluginTopic):
670
629
  not_available_info = str(e)
671
630
 
672
631
  if datetime_now >= product.next_try and datetime_now < stop_time:
673
- wait_seconds = (
632
+ wait_seconds: Union[float, int] = (
674
633
  datetime_now - product.next_try + timedelta(minutes=wait)
675
634
  ).seconds
676
635
  retry_count += 1