eodag 3.0.1__py3-none-any.whl → 3.1.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (87) hide show
  1. eodag/api/core.py +174 -138
  2. eodag/api/product/_assets.py +44 -15
  3. eodag/api/product/_product.py +58 -47
  4. eodag/api/product/drivers/__init__.py +81 -4
  5. eodag/api/product/drivers/base.py +65 -4
  6. eodag/api/product/drivers/generic.py +65 -0
  7. eodag/api/product/drivers/sentinel1.py +97 -0
  8. eodag/api/product/drivers/sentinel2.py +95 -0
  9. eodag/api/product/metadata_mapping.py +117 -90
  10. eodag/api/search_result.py +13 -23
  11. eodag/cli.py +26 -5
  12. eodag/config.py +86 -92
  13. eodag/plugins/apis/base.py +1 -1
  14. eodag/plugins/apis/ecmwf.py +42 -22
  15. eodag/plugins/apis/usgs.py +17 -16
  16. eodag/plugins/authentication/aws_auth.py +16 -13
  17. eodag/plugins/authentication/base.py +5 -3
  18. eodag/plugins/authentication/header.py +3 -3
  19. eodag/plugins/authentication/keycloak.py +4 -4
  20. eodag/plugins/authentication/oauth.py +7 -3
  21. eodag/plugins/authentication/openid_connect.py +22 -16
  22. eodag/plugins/authentication/sas_auth.py +4 -4
  23. eodag/plugins/authentication/token.py +41 -10
  24. eodag/plugins/authentication/token_exchange.py +1 -1
  25. eodag/plugins/base.py +4 -4
  26. eodag/plugins/crunch/base.py +4 -4
  27. eodag/plugins/crunch/filter_date.py +4 -4
  28. eodag/plugins/crunch/filter_latest_intersect.py +6 -6
  29. eodag/plugins/crunch/filter_latest_tpl_name.py +7 -7
  30. eodag/plugins/crunch/filter_overlap.py +4 -4
  31. eodag/plugins/crunch/filter_property.py +6 -7
  32. eodag/plugins/download/aws.py +146 -87
  33. eodag/plugins/download/base.py +38 -56
  34. eodag/plugins/download/creodias_s3.py +29 -0
  35. eodag/plugins/download/http.py +173 -183
  36. eodag/plugins/download/s3rest.py +10 -11
  37. eodag/plugins/manager.py +10 -20
  38. eodag/plugins/search/__init__.py +6 -5
  39. eodag/plugins/search/base.py +90 -46
  40. eodag/plugins/search/build_search_result.py +1048 -361
  41. eodag/plugins/search/cop_marine.py +22 -12
  42. eodag/plugins/search/creodias_s3.py +9 -73
  43. eodag/plugins/search/csw.py +11 -11
  44. eodag/plugins/search/data_request_search.py +19 -18
  45. eodag/plugins/search/qssearch.py +99 -258
  46. eodag/plugins/search/stac_list_assets.py +85 -0
  47. eodag/plugins/search/static_stac_search.py +4 -4
  48. eodag/resources/ext_product_types.json +1 -1
  49. eodag/resources/product_types.yml +1134 -325
  50. eodag/resources/providers.yml +906 -2006
  51. eodag/resources/stac_api.yml +2 -2
  52. eodag/resources/user_conf_template.yml +10 -9
  53. eodag/rest/cache.py +2 -2
  54. eodag/rest/config.py +3 -3
  55. eodag/rest/core.py +112 -82
  56. eodag/rest/errors.py +5 -5
  57. eodag/rest/server.py +33 -14
  58. eodag/rest/stac.py +41 -38
  59. eodag/rest/types/collections_search.py +3 -3
  60. eodag/rest/types/eodag_search.py +29 -23
  61. eodag/rest/types/queryables.py +42 -31
  62. eodag/rest/types/stac_search.py +15 -25
  63. eodag/rest/utils/__init__.py +14 -21
  64. eodag/rest/utils/cql_evaluate.py +6 -6
  65. eodag/rest/utils/rfc3339.py +2 -2
  66. eodag/types/__init__.py +141 -32
  67. eodag/types/bbox.py +2 -2
  68. eodag/types/download_args.py +3 -3
  69. eodag/types/queryables.py +183 -72
  70. eodag/types/search_args.py +4 -4
  71. eodag/types/whoosh.py +127 -3
  72. eodag/utils/__init__.py +153 -51
  73. eodag/utils/exceptions.py +28 -21
  74. eodag/utils/import_system.py +2 -2
  75. eodag/utils/repr.py +65 -6
  76. eodag/utils/requests.py +13 -13
  77. eodag/utils/rest.py +2 -2
  78. eodag/utils/s3.py +231 -0
  79. eodag/utils/stac_reader.py +10 -10
  80. {eodag-3.0.1.dist-info → eodag-3.1.0.dist-info}/METADATA +77 -76
  81. eodag-3.1.0.dist-info/RECORD +113 -0
  82. {eodag-3.0.1.dist-info → eodag-3.1.0.dist-info}/WHEEL +1 -1
  83. {eodag-3.0.1.dist-info → eodag-3.1.0.dist-info}/entry_points.txt +4 -2
  84. eodag/utils/constraints.py +0 -244
  85. eodag-3.0.1.dist-info/RECORD +0 -109
  86. {eodag-3.0.1.dist-info → eodag-3.1.0.dist-info}/LICENSE +0 -0
  87. {eodag-3.0.1.dist-info → eodag-3.1.0.dist-info}/top_level.txt +0 -0
@@ -27,17 +27,8 @@ from datetime import datetime
27
27
  from email.message import Message
28
28
  from itertools import chain
29
29
  from json import JSONDecodeError
30
- from typing import (
31
- TYPE_CHECKING,
32
- Any,
33
- Dict,
34
- Iterator,
35
- List,
36
- Optional,
37
- TypedDict,
38
- Union,
39
- cast,
40
- )
30
+ from pathlib import Path
31
+ from typing import TYPE_CHECKING, Any, Iterator, Optional, TypedDict, Union, cast
41
32
  from urllib.parse import parse_qs, urlparse
42
33
 
43
34
  import geojson
@@ -71,6 +62,7 @@ from eodag.utils import (
71
62
  guess_file_type,
72
63
  parse_header,
73
64
  path_to_uri,
65
+ rename_with_version,
74
66
  sanitize,
75
67
  string_to_jsonpath,
76
68
  uri_to_path,
@@ -80,8 +72,8 @@ from eodag.utils.exceptions import (
80
72
  DownloadError,
81
73
  MisconfiguredError,
82
74
  NotAvailableError,
83
- RequestError,
84
75
  TimeOutError,
76
+ ValidationError,
85
77
  )
86
78
 
87
79
  if TYPE_CHECKING:
@@ -90,6 +82,7 @@ if TYPE_CHECKING:
90
82
  from eodag.api.product import Asset, EOProduct # type: ignore
91
83
  from eodag.api.search_result import SearchResult
92
84
  from eodag.config import PluginConfig
85
+ from eodag.types import S3SessionKwargs
93
86
  from eodag.types.download_args import DownloadConf
94
87
  from eodag.utils import DownloadedCallback, Unpack
95
88
 
@@ -110,7 +103,7 @@ class HTTPDownload(Download):
110
103
  extracted; default: ``True``
111
104
  * :attr:`~eodag.config.PluginConfig.auth_error_code` (``int``): which error code is returned in case of an
112
105
  authentication error
113
- * :attr:`~eodag.config.PluginConfig.dl_url_params` (``Dict[str, Any]``): parameters to be
106
+ * :attr:`~eodag.config.PluginConfig.dl_url_params` (``dict[str, Any]``): parameters to be
114
107
  added to the query params of the request
115
108
  * :attr:`~eodag.config.PluginConfig.archive_depth` (``int``): level in extracted path tree where to find data;
116
109
  default: ``1``
@@ -122,8 +115,6 @@ class HTTPDownload(Download):
122
115
  default: ``5``
123
116
  * :attr:`~eodag.config.PluginConfig.ssl_verify` (``bool``): if the ssl certificates should be verified in
124
117
  requests; default: ``True``
125
- * :attr:`~eodag.config.PluginConfig.output_extension` (``str``): which extension should be used for the
126
- downloaded file
127
118
  * :attr:`~eodag.config.PluginConfig.no_auth_download` (``bool``): if the download should be done without
128
119
  authentication; default: ``True``
129
120
  * :attr:`~eodag.config.PluginConfig.order_enabled` (``bool``): if the product has to be ordered to download it;
@@ -131,7 +122,7 @@ class HTTPDownload(Download):
131
122
  the search plugin used for the provider; default: ``False``
132
123
  * :attr:`~eodag.config.PluginConfig.order_method` (``str``): HTTP request method for the order request (``GET``
133
124
  or ``POST``); default: ``GET``
134
- * :attr:`~eodag.config.PluginConfig.order_headers` (``[Dict[str, str]]``): headers to be added to the order
125
+ * :attr:`~eodag.config.PluginConfig.order_headers` (``[dict[str, str]]``): headers to be added to the order
135
126
  request
136
127
  * :attr:`~eodag.config.PluginConfig.order_on_response` (:class:`~eodag.config.PluginConfig.OrderOnResponse`):
137
128
  a typed dictionary containing the key ``metadata_mapping`` which can be used to add new product properties
@@ -139,22 +130,21 @@ class HTTPDownload(Download):
139
130
  * :attr:`~eodag.config.PluginConfig.order_status` (:class:`~eodag.config.PluginConfig.OrderStatus`):
140
131
  configuration to handle the order status; contains information which method to use, how the response data is
141
132
  interpreted, which status corresponds to success, ordered and error and what should be done on success.
142
- * :attr:`~eodag.config.PluginConfig.products` (``Dict[str, Dict[str, Any]``): product type specific config; the
143
- keys are the product types, the values are dictionaries which can contain the keys
144
- :attr:`~eodag.config.PluginConfig.output_extension` and :attr:`~eodag.config.PluginConfig.extract` to
145
- overwrite the provider config for a specific product type
133
+ * :attr:`~eodag.config.PluginConfig.products` (``dict[str, dict[str, Any]``): product type specific config; the
134
+ keys are the product types, the values are dictionaries which can contain the key
135
+ :attr:`~eodag.config.PluginConfig.extract` to overwrite the provider config for a specific product type
146
136
 
147
137
  """
148
138
 
149
139
  def __init__(self, provider: str, config: PluginConfig) -> None:
150
140
  super(HTTPDownload, self).__init__(provider, config)
151
141
 
152
- def order_download(
142
+ def _order(
153
143
  self,
154
144
  product: EOProduct,
155
145
  auth: Optional[AuthBase] = None,
156
146
  **kwargs: Unpack[DownloadConf],
157
- ) -> Optional[Dict[str, Any]]:
147
+ ) -> Optional[dict[str, Any]]:
158
148
  """Send product order request.
159
149
 
160
150
  It will be executed once before the download retry loop, if the product is OFFLINE
@@ -186,7 +176,7 @@ class HTTPDownload(Download):
186
176
  ssl_verify = getattr(self.config, "ssl_verify", True)
187
177
  timeout = getattr(self.config, "timeout", HTTP_REQ_TIMEOUT)
188
178
  OrderKwargs = TypedDict(
189
- "OrderKwargs", {"json": Dict[str, Union[Any, List[str]]]}, total=False
179
+ "OrderKwargs", {"json": dict[str, Union[Any, list[str]]]}, total=False
190
180
  )
191
181
  order_kwargs: OrderKwargs = {}
192
182
  if order_method == "POST":
@@ -226,9 +216,11 @@ class HTTPDownload(Download):
226
216
  product.properties["storageStatus"] = STAGING_STATUS
227
217
  except RequestException as e:
228
218
  self._check_auth_exception(e)
229
- title = product.properties["title"]
230
- message = f"{title} could not be ordered"
231
- raise RequestError.from_error(e, message) from e
219
+ msg = f'{product.properties["title"]} could not be ordered'
220
+ if e.response is not None and e.response.status_code == 400:
221
+ raise ValidationError.from_error(e, msg) from e
222
+ else:
223
+ raise DownloadError.from_error(e, msg) from e
232
224
 
233
225
  return self.order_response_process(response, product)
234
226
  except requests.exceptions.Timeout as exc:
@@ -236,7 +228,7 @@ class HTTPDownload(Download):
236
228
 
237
229
  def order_response_process(
238
230
  self, response: Response, product: EOProduct
239
- ) -> Optional[Dict[str, Any]]:
231
+ ) -> Optional[dict[str, Any]]:
240
232
  """Process order response
241
233
 
242
234
  :param response: The order response
@@ -271,7 +263,7 @@ class HTTPDownload(Download):
271
263
 
272
264
  return json_response
273
265
 
274
- def order_download_status(
266
+ def _order_status(
275
267
  self,
276
268
  product: EOProduct,
277
269
  auth: Optional[AuthBase] = None,
@@ -300,7 +292,7 @@ class HTTPDownload(Download):
300
292
  def _request(
301
293
  url: str,
302
294
  method: str = "GET",
303
- headers: Optional[Dict[str, Any]] = None,
295
+ headers: Optional[dict[str, Any]] = None,
304
296
  json: Optional[Any] = None,
305
297
  timeout: int = HTTP_REQ_TIMEOUT,
306
298
  ) -> Response:
@@ -336,7 +328,7 @@ class HTTPDownload(Download):
336
328
  except requests.exceptions.Timeout as exc:
337
329
  raise TimeOutError(exc, timeout=timeout) from exc
338
330
 
339
- status_request: Dict[str, Any] = status_config.get("request", {})
331
+ status_request: dict[str, Any] = status_config.get("request", {})
340
332
  status_request_method = str(status_request.get("method", "GET")).upper()
341
333
 
342
334
  if status_request_method == "POST":
@@ -353,8 +345,8 @@ class HTTPDownload(Download):
353
345
 
354
346
  # check header for success before full status request
355
347
  skip_parsing_status_response = False
356
- status_dict: Dict[str, Any] = {}
357
- config_on_success: Dict[str, Any] = status_config.get("on_success", {})
348
+ status_dict: dict[str, Any] = {}
349
+ config_on_success: dict[str, Any] = status_config.get("on_success", {})
358
350
  on_success_mm = config_on_success.get("metadata_mapping", {})
359
351
 
360
352
  status_response_content_needed = (
@@ -398,13 +390,11 @@ class HTTPDownload(Download):
398
390
  # success and no need to get status response content
399
391
  skip_parsing_status_response = True
400
392
  except RequestException as e:
401
- raise DownloadError(
402
- "%s order status could not be checked, request returned %s"
403
- % (
404
- product.properties["title"],
405
- e,
406
- )
407
- ) from e
393
+ msg = f'{product.properties["title"]} order status could not be checked'
394
+ if e.response is not None and e.response.status_code == 400:
395
+ raise ValidationError.from_error(e, msg) from e
396
+ else:
397
+ raise DownloadError.from_error(e, msg) from e
408
398
 
409
399
  if not skip_parsing_status_response:
410
400
  # status request
@@ -440,13 +430,13 @@ class HTTPDownload(Download):
440
430
  product.properties["orderStatus"] = status_dict.get("status")
441
431
 
442
432
  # handle status error
443
- errors: Dict[str, Any] = status_config.get("error", {})
433
+ errors: dict[str, Any] = status_config.get("error", {})
444
434
  if errors and errors.items() <= status_dict.items():
445
435
  raise DownloadError(
446
436
  f"Provider {product.provider} returned: {status_dict.get('error_message', status_message)}"
447
437
  )
448
438
 
449
- success_status: Dict[str, Any] = status_config.get("success", {}).get("status")
439
+ success_status: dict[str, Any] = status_config.get("success", {}).get("status")
450
440
  # if not success
451
441
  if (success_status and success_status != status_dict.get("status")) or (
452
442
  success_code and success_code != response.status_code
@@ -564,10 +554,10 @@ class HTTPDownload(Download):
564
554
  def download(
565
555
  self,
566
556
  product: EOProduct,
567
- auth: Optional[Union[AuthBase, Dict[str, str]]] = None,
557
+ auth: Optional[Union[AuthBase, S3SessionKwargs]] = None,
568
558
  progress_callback: Optional[ProgressCallback] = None,
569
- wait: int = DEFAULT_DOWNLOAD_WAIT,
570
- timeout: int = DEFAULT_DOWNLOAD_TIMEOUT,
559
+ wait: float = DEFAULT_DOWNLOAD_WAIT,
560
+ timeout: float = DEFAULT_DOWNLOAD_TIMEOUT,
571
561
  **kwargs: Unpack[DownloadConf],
572
562
  ) -> Optional[str]:
573
563
  """Download a product using HTTP protocol.
@@ -585,13 +575,6 @@ class HTTPDownload(Download):
585
575
  )
586
576
  progress_callback = ProgressCallback(disable=True)
587
577
 
588
- output_extension = getattr(self.config, "products", {}).get(
589
- product.product_type, {}
590
- ).get("output_extension", None) or getattr(
591
- self.config, "output_extension", ".zip"
592
- )
593
- kwargs["output_extension"] = kwargs.get("output_extension", output_extension)
594
-
595
578
  fs_path, record_filename = self._prepare_download(
596
579
  product,
597
580
  progress_callback=progress_callback,
@@ -610,7 +593,7 @@ class HTTPDownload(Download):
610
593
  try:
611
594
  fs_path = self._download_assets(
612
595
  product,
613
- fs_path.replace(".zip", ""),
596
+ fs_path,
614
597
  record_filename,
615
598
  auth,
616
599
  progress_callback,
@@ -627,82 +610,64 @@ class HTTPDownload(Download):
627
610
 
628
611
  url = product.remote_location
629
612
 
630
- @self._download_retry(product, wait, timeout)
613
+ @self._order_download_retry(product, wait, timeout)
631
614
  def download_request(
632
615
  product: EOProduct,
633
616
  auth: AuthBase,
634
617
  progress_callback: ProgressCallback,
635
- wait: int,
636
- timeout: int,
618
+ wait: float,
619
+ timeout: float,
637
620
  **kwargs: Unpack[DownloadConf],
638
- ) -> None:
639
- chunks = self._stream_download(product, auth, progress_callback, **kwargs)
621
+ ) -> os.PathLike:
640
622
  is_empty = True
623
+ chunk_iterator = self._stream_download(
624
+ product, auth, progress_callback, **kwargs
625
+ )
626
+ if fs_path is not None:
627
+ ext = Path(product.filename).suffix
628
+ path = Path(fs_path).with_suffix(ext)
641
629
 
642
- with open(fs_path, "wb") as fhandle:
643
- for chunk in chunks:
644
- is_empty = False
645
- fhandle.write(chunk)
630
+ with open(path, "wb") as fhandle:
631
+ for chunk in chunk_iterator:
632
+ is_empty = False
633
+ progress_callback(len(chunk))
634
+ fhandle.write(chunk)
635
+ self.stream.close() # Closing response stream
646
636
 
647
- if is_empty:
648
- raise DownloadError(f"product {product.properties['id']} is empty")
637
+ if is_empty:
638
+ raise DownloadError(f"product {product.properties['id']} is empty")
649
639
 
650
- download_request(product, auth, progress_callback, wait, timeout, **kwargs)
640
+ return path
641
+ else:
642
+ raise DownloadError(
643
+ f"download of product {product.properties['id']} failed"
644
+ )
645
+
646
+ path = download_request(
647
+ product, auth, progress_callback, wait, timeout, **kwargs
648
+ )
651
649
 
652
650
  with open(record_filename, "w") as fh:
653
651
  fh.write(url)
654
652
  logger.debug("Download recorded in %s", record_filename)
655
653
 
656
- # Check that the downloaded file is really a zip file
657
- if not zipfile.is_zipfile(fs_path) and output_extension == ".zip":
658
- logger.warning(
659
- "Downloaded product is not a Zip File. Please check its file type before using it"
660
- )
661
- new_fs_path = os.path.join(
662
- os.path.dirname(fs_path),
663
- sanitize(product.properties["title"]),
664
- )
665
- if os.path.isfile(fs_path) and not tarfile.is_tarfile(fs_path):
666
- if not os.path.isdir(new_fs_path):
667
- os.makedirs(new_fs_path)
668
- shutil.move(fs_path, new_fs_path)
669
- file_path = os.path.join(new_fs_path, os.path.basename(fs_path))
670
- new_file_path = file_path[: file_path.index(".zip")]
671
- shutil.move(file_path, new_file_path)
672
- # in the case where the outputs extension has not been set
673
- # to ".tar" in the product type nor provider configuration
674
- elif tarfile.is_tarfile(fs_path):
675
- if not new_fs_path.endswith(".tar"):
676
- new_fs_path += ".tar"
677
- shutil.move(fs_path, new_fs_path)
678
- kwargs["output_extension"] = ".tar"
679
- product_path = self._finalize(
680
- new_fs_path,
681
- progress_callback=progress_callback,
682
- **kwargs,
683
- )
684
- product.location = path_to_uri(product_path)
685
- return product_path
686
- else:
687
- # not a file (dir with zip extension)
688
- shutil.move(fs_path, new_fs_path)
689
- product.location = path_to_uri(new_fs_path)
690
- return new_fs_path
691
-
692
- if os.path.isfile(fs_path) and not (
693
- zipfile.is_zipfile(fs_path) or tarfile.is_tarfile(fs_path)
654
+ if os.path.isfile(path) and not (
655
+ zipfile.is_zipfile(path) or tarfile.is_tarfile(path)
694
656
  ):
695
657
  new_fs_path = os.path.join(
696
- os.path.dirname(fs_path),
658
+ os.path.dirname(path),
697
659
  sanitize(product.properties["title"]),
698
660
  )
661
+ if os.path.isfile(new_fs_path):
662
+ rename_with_version(new_fs_path)
699
663
  if not os.path.isdir(new_fs_path):
700
664
  os.makedirs(new_fs_path)
701
- shutil.move(fs_path, new_fs_path)
665
+ shutil.move(path, new_fs_path)
702
666
  product.location = path_to_uri(new_fs_path)
703
667
  return new_fs_path
668
+
704
669
  product_path = self._finalize(
705
- fs_path,
670
+ str(path),
706
671
  progress_callback=progress_callback,
707
672
  **kwargs,
708
673
  )
@@ -743,24 +708,15 @@ class HTTPDownload(Download):
743
708
  ext = guess_extension(content_type)
744
709
  if ext:
745
710
  filename += ext
746
- else:
747
- output_extension: Optional[str] = (
748
- getattr(self.config, "products", {})
749
- .get(product.product_type, {})
750
- .get("output_extension")
751
- )
752
- if output_extension:
753
- filename += output_extension
754
-
755
711
  return filename
756
712
 
757
713
  def _stream_download_dict(
758
714
  self,
759
715
  product: EOProduct,
760
- auth: Optional[Union[AuthBase, Dict[str, str]]] = None,
716
+ auth: Optional[Union[AuthBase, S3SessionKwargs]] = None,
761
717
  progress_callback: Optional[ProgressCallback] = None,
762
- wait: int = DEFAULT_DOWNLOAD_WAIT,
763
- timeout: int = DEFAULT_DOWNLOAD_TIMEOUT,
718
+ wait: float = DEFAULT_DOWNLOAD_WAIT,
719
+ timeout: float = DEFAULT_DOWNLOAD_TIMEOUT,
764
720
  **kwargs: Unpack[DownloadConf],
765
721
  ) -> StreamResponse:
766
722
  r"""
@@ -816,13 +772,17 @@ class HTTPDownload(Download):
816
772
  )
817
773
 
818
774
  else:
775
+ # get first chunk to check if it does not contain an error (if it does, that error will be raised)
776
+ first_chunks_tuple = next(chunks_tuples)
819
777
  outputs_filename = (
820
778
  sanitize(product.properties["title"])
821
779
  if "title" in product.properties
822
780
  else sanitize(product.properties.get("id", "download"))
823
781
  )
824
782
  return StreamResponse(
825
- content=stream_zip(chunks_tuples),
783
+ content=stream_zip(
784
+ chain(iter([first_chunks_tuple]), chunks_tuples)
785
+ ),
826
786
  media_type="application/zip",
827
787
  headers={
828
788
  "content-disposition": f"attachment; filename={outputs_filename}.zip",
@@ -834,17 +794,20 @@ class HTTPDownload(Download):
834
794
  else:
835
795
  pass
836
796
 
837
- chunks = self._stream_download(product, auth, progress_callback, **kwargs)
797
+ chunk_iterator = self._stream_download(
798
+ product, auth, progress_callback, **kwargs
799
+ )
800
+
838
801
  # start reading chunks to set product.headers
839
802
  try:
840
- first_chunk = next(chunks)
803
+ first_chunk = next(chunk_iterator)
841
804
  except StopIteration:
842
805
  # product is empty file
843
806
  logger.error("product %s is empty", product.properties["id"])
844
807
  raise NotAvailableError(f"product {product.properties['id']} is empty")
845
808
 
846
809
  return StreamResponse(
847
- content=chain(iter([first_chunk]), chunks),
810
+ content=chain(iter([first_chunk]), chunk_iterator),
848
811
  headers=product.headers,
849
812
  )
850
813
 
@@ -902,6 +865,44 @@ class HTTPDownload(Download):
902
865
  else:
903
866
  logger.error("Error while getting resource :\n%s", tb.format_exc())
904
867
 
868
+ def _order_request(
869
+ self,
870
+ product: EOProduct,
871
+ auth: Optional[AuthBase],
872
+ ) -> None:
873
+ if (
874
+ "orderLink" in product.properties
875
+ and product.properties.get("storageStatus") == OFFLINE_STATUS
876
+ and not product.properties.get("orderStatus")
877
+ ):
878
+ self._order(product=product, auth=auth)
879
+
880
+ if (
881
+ product.properties.get("orderStatusLink", None)
882
+ and product.properties.get("storageStatus") != ONLINE_STATUS
883
+ ):
884
+ self._order_status(product=product, auth=auth)
885
+
886
+ def order(
887
+ self,
888
+ product: EOProduct,
889
+ auth: Optional[Union[AuthBase, S3SessionKwargs]] = None,
890
+ wait: float = DEFAULT_DOWNLOAD_WAIT,
891
+ timeout: float = DEFAULT_DOWNLOAD_TIMEOUT,
892
+ ) -> None:
893
+ """
894
+ Order product and poll to check its status
895
+
896
+ :param product: The EO product to download
897
+ :param auth: (optional) authenticated object
898
+ :param wait: (optional) Wait time in minutes between two order status check
899
+ :param timeout: (optional) Maximum time in minutes before stop checking
900
+ order status
901
+ """
902
+ self._order_download_retry(product, wait, timeout)(self._order_request)(
903
+ product, auth
904
+ )
905
+
905
906
  def _stream_download(
906
907
  self,
907
908
  product: EOProduct,
@@ -910,8 +911,9 @@ class HTTPDownload(Download):
910
911
  **kwargs: Unpack[DownloadConf],
911
912
  ) -> Iterator[Any]:
912
913
  """
913
- fetches a zip file containing the assets of a given product as a stream
914
+ Fetches a zip file containing the assets of a given product as a stream
914
915
  and returns a generator yielding the chunks of the file
916
+
915
917
  :param product: product for which the assets should be downloaded
916
918
  :param auth: The configuration of a plugin of type Authentication
917
919
  :param progress_callback: A method or a callable object
@@ -928,18 +930,8 @@ class HTTPDownload(Download):
928
930
  ssl_verify = getattr(self.config, "ssl_verify", True)
929
931
 
930
932
  ordered_message = ""
931
- if (
932
- "orderLink" in product.properties
933
- and product.properties.get("storageStatus") == OFFLINE_STATUS
934
- and not product.properties.get("orderStatus")
935
- ):
936
- self.order_download(product=product, auth=auth)
937
-
938
- if (
939
- product.properties.get("orderStatusLink", None)
940
- and product.properties.get("storageStatus") != ONLINE_STATUS
941
- ):
942
- self.order_download_status(product=product, auth=auth)
933
+ # retry handled at download level
934
+ self._order_request(product, auth)
943
935
 
944
936
  params = kwargs.pop("dl_url_params", None) or getattr(
945
937
  self.config, "dl_url_params", {}
@@ -957,7 +949,7 @@ class HTTPDownload(Download):
957
949
  if not query_dict and parts.query:
958
950
  query_dict = geojson.loads(parts.query)
959
951
  req_url = parts._replace(query="").geturl()
960
- req_kwargs: Dict[str, Any] = {"json": query_dict} if query_dict else {}
952
+ req_kwargs: dict[str, Any] = {"json": query_dict} if query_dict else {}
961
953
  else:
962
954
  req_url = url
963
955
  req_kwargs = {}
@@ -969,7 +961,7 @@ class HTTPDownload(Download):
969
961
  auth = None
970
962
 
971
963
  s = requests.Session()
972
- with s.request(
964
+ self.stream = s.request(
973
965
  req_method,
974
966
  req_url,
975
967
  stream=True,
@@ -979,55 +971,53 @@ class HTTPDownload(Download):
979
971
  timeout=DEFAULT_STREAM_REQUESTS_TIMEOUT,
980
972
  verify=ssl_verify,
981
973
  **req_kwargs,
982
- ) as self.stream:
983
- try:
984
- self.stream.raise_for_status()
985
- except requests.exceptions.Timeout as exc:
986
- raise TimeOutError(
987
- exc, timeout=DEFAULT_STREAM_REQUESTS_TIMEOUT
988
- ) from exc
989
- except RequestException as e:
990
- self._process_exception(e, product, ordered_message)
991
- else:
992
- # check if product was ordered
993
-
994
- if getattr(
995
- self.stream, "status_code", None
996
- ) is not None and self.stream.status_code == getattr(
997
- self.config, "order_status", {}
998
- ).get(
999
- "ordered", {}
1000
- ).get(
1001
- "http_code"
1002
- ):
1003
- product.properties["storageStatus"] = "ORDERED"
1004
- self._process_exception(None, product, ordered_message)
1005
- stream_size = self._check_stream_size(product) or None
1006
-
1007
- product.headers = self.stream.headers
1008
- filename = self._check_product_filename(product) or None
1009
- product.headers[
1010
- "content-disposition"
1011
- ] = f"attachment; filename={filename}"
1012
- content_type = product.headers.get("Content-Type")
1013
- guessed_content_type = (
1014
- guess_file_type(filename) if filename and not content_type else None
1015
- )
1016
- if guessed_content_type is not None:
1017
- product.headers["Content-Type"] = guessed_content_type
974
+ )
975
+ try:
976
+ self.stream.raise_for_status()
977
+ except requests.exceptions.Timeout as exc:
978
+ raise TimeOutError(exc, timeout=DEFAULT_STREAM_REQUESTS_TIMEOUT) from exc
979
+ except RequestException as e:
980
+ self._process_exception(e, product, ordered_message)
981
+ raise DownloadError(
982
+ f"download of {product.properties['id']} is empty"
983
+ ) from e
984
+ else:
985
+ # check if product was ordered
986
+
987
+ if getattr(
988
+ self.stream, "status_code", None
989
+ ) is not None and self.stream.status_code == getattr(
990
+ self.config, "order_status", {}
991
+ ).get(
992
+ "ordered", {}
993
+ ).get(
994
+ "http_code"
995
+ ):
996
+ product.properties["storageStatus"] = "ORDERED"
997
+ self._process_exception(None, product, ordered_message)
998
+ stream_size = self._check_stream_size(product) or None
999
+
1000
+ product.headers = self.stream.headers
1001
+ filename = self._check_product_filename(product)
1002
+ product.headers["content-disposition"] = f"attachment; filename={filename}"
1003
+ content_type = product.headers.get("Content-Type")
1004
+ guessed_content_type = (
1005
+ guess_file_type(filename) if filename and not content_type else None
1006
+ )
1007
+ if guessed_content_type is not None:
1008
+ product.headers["Content-Type"] = guessed_content_type
1018
1009
 
1019
- progress_callback.reset(total=stream_size)
1020
- for chunk in self.stream.iter_content(chunk_size=64 * 1024):
1021
- if chunk:
1022
- progress_callback(len(chunk))
1023
- yield chunk
1010
+ progress_callback.reset(total=stream_size)
1011
+
1012
+ product.filename = filename
1013
+ return self.stream.iter_content(chunk_size=64 * 1024)
1024
1014
 
1025
1015
  def _stream_download_assets(
1026
1016
  self,
1027
1017
  product: EOProduct,
1028
1018
  auth: Optional[AuthBase] = None,
1029
1019
  progress_callback: Optional[ProgressCallback] = None,
1030
- assets_values: List[Asset] = [],
1020
+ assets_values: list[Asset] = [],
1031
1021
  **kwargs: Unpack[DownloadConf],
1032
1022
  ) -> Iterator[Any]:
1033
1023
  if progress_callback is None:
@@ -1297,9 +1287,9 @@ class HTTPDownload(Download):
1297
1287
 
1298
1288
  def _get_asset_sizes(
1299
1289
  self,
1300
- assets_values: List[Asset],
1290
+ assets_values: list[Asset],
1301
1291
  auth: Optional[AuthBase],
1302
- params: Optional[Dict[str, str]],
1292
+ params: Optional[dict[str, str]],
1303
1293
  zipped: bool = False,
1304
1294
  ) -> int:
1305
1295
  total_size = 0
@@ -1372,11 +1362,11 @@ class HTTPDownload(Download):
1372
1362
  def download_all(
1373
1363
  self,
1374
1364
  products: SearchResult,
1375
- auth: Optional[Union[AuthBase, Dict[str, str]]] = None,
1365
+ auth: Optional[Union[AuthBase, S3SessionKwargs]] = None,
1376
1366
  downloaded_callback: Optional[DownloadedCallback] = None,
1377
1367
  progress_callback: Optional[ProgressCallback] = None,
1378
- wait: int = DEFAULT_DOWNLOAD_WAIT,
1379
- timeout: int = DEFAULT_DOWNLOAD_TIMEOUT,
1368
+ wait: float = DEFAULT_DOWNLOAD_WAIT,
1369
+ timeout: float = DEFAULT_DOWNLOAD_TIMEOUT,
1380
1370
  **kwargs: Unpack[DownloadConf],
1381
1371
  ):
1382
1372
  """