eodag 3.1.0b2__py3-none-any.whl → 3.2.1__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
eodag/api/core.py CHANGED
@@ -23,13 +23,13 @@ import os
23
23
  import re
24
24
  import shutil
25
25
  import tempfile
26
+ from importlib.metadata import version
27
+ from importlib.resources import files as res_files
26
28
  from operator import itemgetter
27
29
  from typing import TYPE_CHECKING, Any, Iterator, Optional, Union
28
30
 
29
31
  import geojson
30
- import pkg_resources
31
32
  import yaml.parser
32
- from pkg_resources import resource_filename
33
33
  from whoosh import analysis, fields
34
34
  from whoosh.fields import Schema
35
35
  from whoosh.index import exists_in, open_dir
@@ -119,8 +119,8 @@ class EODataAccessGateway:
119
119
  user_conf_file_path: Optional[str] = None,
120
120
  locations_conf_path: Optional[str] = None,
121
121
  ) -> None:
122
- product_types_config_path = resource_filename(
123
- "eodag", os.path.join("resources/", "product_types.yml")
122
+ product_types_config_path = os.getenv("EODAG_PRODUCT_TYPES_CFG_FILE") or str(
123
+ res_files("eodag") / "resources" / "product_types.yml"
124
124
  )
125
125
  self.product_types_config = SimpleYamlProxyConfig(product_types_config_path)
126
126
  self.product_types_config_md5 = obj_md5sum(self.product_types_config.source)
@@ -161,8 +161,8 @@ class EODataAccessGateway:
161
161
  user_conf_file_path = standard_configuration_path
162
162
  if not os.path.isfile(standard_configuration_path):
163
163
  shutil.copy(
164
- resource_filename(
165
- "eodag", os.path.join("resources", "user_conf_template.yml")
164
+ str(
165
+ res_files("eodag") / "resources" / "user_conf_template.yml"
166
166
  ),
167
167
  standard_configuration_path,
168
168
  )
@@ -203,9 +203,8 @@ class EODataAccessGateway:
203
203
  locations_conf_path = os.path.join(self.conf_dir, "locations.yml")
204
204
  if not os.path.isfile(locations_conf_path):
205
205
  # copy locations conf file and replace path example
206
- locations_conf_template = resource_filename(
207
- "eodag",
208
- os.path.join("resources", "locations_conf_template.yml"),
206
+ locations_conf_template = str(
207
+ res_files("eodag") / "resources" / "locations_conf_template.yml"
209
208
  )
210
209
  with (
211
210
  open(locations_conf_template) as infile,
@@ -222,14 +221,14 @@ class EODataAccessGateway:
222
221
  outfile.write(line)
223
222
  # copy sample shapefile dir
224
223
  shutil.copytree(
225
- resource_filename("eodag", os.path.join("resources", "shp")),
224
+ str(res_files("eodag") / "resources" / "shp"),
226
225
  os.path.join(self.conf_dir, "shp"),
227
226
  )
228
227
  self.set_locations_conf(locations_conf_path)
229
228
 
230
229
  def get_version(self) -> str:
231
230
  """Get eodag package version"""
232
- return pkg_resources.get_distribution("eodag").version
231
+ return version("eodag")
233
232
 
234
233
  def build_index(self) -> None:
235
234
  """Build a `Whoosh <https://whoosh.readthedocs.io/en/latest/index.html>`_
@@ -31,12 +31,27 @@ if TYPE_CHECKING:
31
31
 
32
32
 
33
33
  class AssetsDict(UserDict):
34
- """A UserDict object listing assets contained in a
35
- :class:`~eodag.api.product._product.EOProduct` resulting from a search.
34
+ """A UserDict object which values are :class:`~eodag.api.product._assets.Asset`
35
+ contained in a :class:`~eodag.api.product._product.EOProduct` resulting from a
36
+ search.
36
37
 
37
38
  :param product: Product resulting from a search
38
39
  :param args: (optional) Arguments used to init the dictionary
39
40
  :param kwargs: (optional) Additional named-arguments used to init the dictionary
41
+
42
+ Example
43
+ -------
44
+
45
+ >>> from eodag.api.product import EOProduct
46
+ >>> product = EOProduct(
47
+ ... provider="foo",
48
+ ... properties={"id": "bar", "geometry": "POINT (0 0)"}
49
+ ... )
50
+ >>> type(product.assets)
51
+ <class 'eodag.api.product._assets.AssetsDict'>
52
+ >>> product.assets.update({"foo": {"href": "http://somewhere/something"}})
53
+ >>> product.assets
54
+ {'foo': {'href': 'http://somewhere/something'}}
40
55
  """
41
56
 
42
57
  product: EOProduct
@@ -56,22 +71,29 @@ class AssetsDict(UserDict):
56
71
  """
57
72
  return {k: v.as_dict() for k, v in self.data.items()}
58
73
 
59
- def get_values(self, asset_filter: str = "") -> list[Asset]:
74
+ def get_values(self, asset_filter: str = "", regex=True) -> list[Asset]:
60
75
  """
61
76
  retrieves the assets matching the given filter
62
77
 
63
78
  :param asset_filter: regex filter with which the assets should be matched
79
+ :param regex: Uses regex to match the asset key or simply compare strings
64
80
  :return: list of assets
65
81
  """
66
82
  if asset_filter:
67
- filter_regex = re.compile(asset_filter)
68
- assets_keys = list(self.keys())
69
- assets_keys = list(filter(filter_regex.fullmatch, assets_keys))
83
+ if regex:
84
+ filter_regex = re.compile(asset_filter)
85
+ assets_keys = list(self.keys())
86
+ assets_keys = list(filter(filter_regex.fullmatch, assets_keys))
87
+ else:
88
+ assets_keys = [a for a in self.keys() if a == asset_filter]
70
89
  filtered_assets = {}
71
90
  if len(assets_keys) > 0:
72
91
  filtered_assets = {a_key: self.get(a_key) for a_key in assets_keys}
73
92
  assets_values = [a for a in filtered_assets.values() if a and "href" in a]
74
- if not assets_values:
93
+ if not assets_values and regex:
94
+ # retry without regex
95
+ return self.get_values(asset_filter, regex=False)
96
+ elif not assets_values:
75
97
  raise NotAvailableError(
76
98
  rf"No asset key matching re.fullmatch(r'{asset_filter}') was found in {self.product}"
77
99
  )
@@ -119,13 +141,27 @@ class AssetsDict(UserDict):
119
141
 
120
142
 
121
143
  class Asset(UserDict):
122
- """A UserDict object containg one of the assets of a
123
- :class:`~eodag.api.product._product.EOProduct` resulting from a search.
144
+ """A UserDict object containg one of the
145
+ :attr:`~eodag.api.product._product.EOProduct.assets` resulting from a search.
124
146
 
125
147
  :param product: Product resulting from a search
126
148
  :param key: asset key
127
149
  :param args: (optional) Arguments used to init the dictionary
128
150
  :param kwargs: (optional) Additional named-arguments used to init the dictionary
151
+
152
+ Example
153
+ -------
154
+
155
+ >>> from eodag.api.product import EOProduct
156
+ >>> product = EOProduct(
157
+ ... provider="foo",
158
+ ... properties={"id": "bar", "geometry": "POINT (0 0)"}
159
+ ... )
160
+ >>> product.assets.update({"foo": {"href": "http://somewhere/something"}})
161
+ >>> type(product.assets["foo"])
162
+ <class 'eodag.api.product._assets.Asset'>
163
+ >>> product.assets["foo"]
164
+ {'href': 'http://somewhere/something'}
129
165
  """
130
166
 
131
167
  product: EOProduct
@@ -89,20 +89,6 @@ class EOProduct:
89
89
 
90
90
  :param provider: The provider from which the product originates
91
91
  :param properties: The metadata of the product
92
- :ivar product_type: The product type
93
- :vartype product_type: str
94
- :ivar location: The path to the product, either remote or local if downloaded
95
- :vartype location: str
96
- :ivar remote_location: The remote path to the product
97
- :vartype remote_location: str
98
- :ivar search_kwargs: The search kwargs used by eodag to search for the product
99
- :vartype search_kwargs: Any
100
- :ivar geometry: The geometry of the product
101
- :vartype geometry: :class:`shapely.geometry.base.BaseGeometry`
102
- :ivar search_intersection: The intersection between the product's geometry
103
- and the search area.
104
- :vartype search_intersection: :class:`shapely.geometry.base.BaseGeometry` or None
105
-
106
92
 
107
93
  .. note::
108
94
  The geojson spec `enforces <https://github.com/geojson/draft-geojson/pull/6>`_
@@ -112,18 +98,28 @@ class EOProduct:
112
98
  mentioned CRS.
113
99
  """
114
100
 
101
+ #: The provider from which the product originates
115
102
  provider: str
103
+ #: The metadata of the product
116
104
  properties: dict[str, Any]
105
+ #: The product type
117
106
  product_type: Optional[str]
118
- location: str
119
- filename: str
120
- remote_location: str
121
- search_kwargs: Any
107
+ #: The geometry of the product
122
108
  geometry: BaseGeometry
109
+ #: The intersection between the product's geometry and the search area.
123
110
  search_intersection: Optional[BaseGeometry]
111
+ #: The path to the product, either remote or local if downloaded
112
+ location: str
113
+ #: The remote path to the product
114
+ remote_location: str
115
+ #: Assets of the product
124
116
  assets: AssetsDict
125
117
  #: Driver enables additional methods to be called on the EOProduct
126
118
  driver: DatasetDriver
119
+ #: Product data filename, stored during download
120
+ filename: str
121
+ #: Product search keyword arguments, stored during search
122
+ search_kwargs: Any
127
123
 
128
124
  def __init__(
129
125
  self, provider: str, properties: dict[str, Any], **kwargs: Any
@@ -49,6 +49,7 @@ from eodag.utils import (
49
49
  get_timestamp,
50
50
  items_recursive_apply,
51
51
  nested_pairs2dict,
52
+ sanitize,
52
53
  string_to_jsonpath,
53
54
  update_nested_dict,
54
55
  )
@@ -176,6 +177,7 @@ def format_metadata(search_param: str, *args: Any, **kwargs: Any) -> str:
176
177
  - ``split_corine_id``: get the product type by splitting the product id
177
178
  - ``to_datetime_dict``: convert a datetime string to a dictionary where values are either a string or a list
178
179
  - ``get_ecmwf_time``: get the time of a datetime string in the ECMWF format
180
+ - ``sanitize``: sanitize string
179
181
 
180
182
  :param search_param: The string to be formatted
181
183
  :param args: (optional) Additional arguments to use in the formatting process
@@ -367,8 +369,8 @@ def format_metadata(search_param: str, *args: Any, **kwargs: Any) -> str:
367
369
  )
368
370
 
369
371
  @staticmethod
370
- def convert_to_geojson(string: str) -> str:
371
- return geojson.dumps(string)
372
+ def convert_to_geojson(value: Any) -> str:
373
+ return geojson.dumps(value)
372
374
 
373
375
  @staticmethod
374
376
  def convert_from_ewkt(ewkt_string: str) -> Union[BaseGeometry, str]:
@@ -496,9 +498,16 @@ def format_metadata(search_param: str, *args: Any, **kwargs: Any) -> str:
496
498
  return NOT_AVAILABLE
497
499
 
498
500
  @staticmethod
499
- def convert_replace_str(string: str, args: str) -> str:
501
+ def convert_replace_str(value: Any, args: str) -> str:
502
+ if isinstance(value, dict):
503
+ value = MetadataFormatter.convert_to_geojson(value)
504
+ elif not isinstance(value, str):
505
+ raise TypeError(
506
+ f"convert_replace_str expects a string or a dict (apply to_geojson). Got {type(value)}"
507
+ )
508
+
500
509
  old, new = ast.literal_eval(args)
501
- return re.sub(old, new, string)
510
+ return re.sub(old, new, value)
502
511
 
503
512
  @staticmethod
504
513
  def convert_recursive_sub_str(
@@ -816,6 +825,11 @@ def format_metadata(search_param: str, *args: Any, **kwargs: Any) -> str:
816
825
  + ":00"
817
826
  ]
818
827
 
828
+ @staticmethod
829
+ def convert_sanitize(text: str) -> str:
830
+ """Sanitize string"""
831
+ return sanitize(text)
832
+
819
833
  @staticmethod
820
834
  def convert_get_dates_from_string(text: str, split_param="-"):
821
835
  reg = "[0-9]{8}" + split_param + "[0-9]{8}"
@@ -1487,7 +1501,11 @@ def get_queryable_from_provider(
1487
1501
  ind = mapping_values.index(provider_queryable)
1488
1502
  return Queryables.get_queryable_from_alias(list(metadata_mapping.keys())[ind])
1489
1503
  for param, param_conf in metadata_mapping.items():
1490
- if isinstance(param_conf, list) and re.search(pattern, param_conf[0]):
1504
+ if (
1505
+ isinstance(param_conf, list)
1506
+ and param_conf[0]
1507
+ and re.search(pattern, param_conf[0])
1508
+ ):
1491
1509
  return Queryables.get_queryable_from_alias(param)
1492
1510
  return None
1493
1511
 
eodag/config.py CHANGED
@@ -20,6 +20,7 @@ from __future__ import annotations
20
20
  import logging
21
21
  import os
22
22
  import tempfile
23
+ from importlib.resources import files as res_files
23
24
  from inspect import isclass
24
25
  from typing import (
25
26
  Annotated,
@@ -41,7 +42,6 @@ import yaml.constructor
41
42
  import yaml.parser
42
43
  from annotated_types import Gt
43
44
  from jsonpath_ng import JSONPath
44
- from pkg_resources import resource_filename
45
45
 
46
46
  from eodag.api.product.metadata_mapping import mtd_cfg_as_conversion_and_querypath
47
47
  from eodag.utils import (
@@ -443,6 +443,9 @@ class PluginConfig(yaml.YAMLObject):
443
443
  literal_search_params: dict[str, str]
444
444
  #: :class:`~eodag.plugins.search.qssearch.QueryStringSearch` Characters that should not be quoted in the url params
445
445
  dont_quote: list[str]
446
+ #: :class:`~eodag.plugins.search.qssearch.QueryStringSearch` Guess assets keys using their ``href``.
447
+ #: Use their original key if ``False``
448
+ asset_key_from_href: bool
446
449
  #: :class:`~eodag.plugins.search.qssearch.ODataV4Search` Dict describing free text search request build
447
450
  free_text_search_operations: dict[str, Any]
448
451
  #: :class:`~eodag.plugins.search.qssearch.ODataV4Search` Set to ``True`` if the metadata is not given in the search
@@ -671,9 +674,9 @@ def load_default_config() -> dict[str, ProviderConfig]:
671
674
 
672
675
  :returns: The default provider's configuration
673
676
  """
674
- eodag_providers_cfg_file = os.getenv(
675
- "EODAG_PROVIDERS_CFG_FILE"
676
- ) or resource_filename("eodag", "resources/providers.yml")
677
+ eodag_providers_cfg_file = os.getenv("EODAG_PROVIDERS_CFG_FILE") or str(
678
+ res_files("eodag") / "resources" / "providers.yml"
679
+ )
677
680
  return load_config(eodag_providers_cfg_file)
678
681
 
679
682
 
@@ -779,6 +782,7 @@ def provider_config_init(
779
782
  and provider_config.search.type
780
783
  in [
781
784
  "StacSearch",
785
+ "StacListAssets",
782
786
  "StaticStacSearch",
783
787
  ]
784
788
  ):
@@ -987,9 +991,7 @@ def load_stac_config() -> dict[str, Any]:
987
991
 
988
992
  :returns: The stac configuration
989
993
  """
990
- return load_yml_config(
991
- resource_filename("eodag", os.path.join("resources/", "stac.yml"))
992
- )
994
+ return load_yml_config(str(res_files("eodag") / "resources" / "stac.yml"))
993
995
 
994
996
 
995
997
  def load_stac_api_config() -> dict[str, Any]:
@@ -997,9 +999,7 @@ def load_stac_api_config() -> dict[str, Any]:
997
999
 
998
1000
  :returns: The stac API configuration
999
1001
  """
1000
- return load_yml_config(
1001
- resource_filename("eodag", os.path.join("resources/", "stac_api.yml"))
1002
- )
1002
+ return load_yml_config(str(res_files("eodag") / "resources" / "stac_api.yml"))
1003
1003
 
1004
1004
 
1005
1005
  def load_stac_provider_config() -> dict[str, Any]:
@@ -1008,7 +1008,7 @@ def load_stac_provider_config() -> dict[str, Any]:
1008
1008
  :returns: The stac provider configuration
1009
1009
  """
1010
1010
  return SimpleYamlProxyConfig(
1011
- resource_filename("eodag", os.path.join("resources/", "stac_provider.yml"))
1011
+ str(res_files("eodag") / "resources" / "stac_provider.yml")
1012
1012
  ).source
1013
1013
 
1014
1014
 
@@ -30,11 +30,7 @@ from pydantic.fields import FieldInfo
30
30
  from eodag.plugins.apis.base import Api
31
31
  from eodag.plugins.search import PreparedSearch
32
32
  from eodag.plugins.search.base import Search
33
- from eodag.plugins.search.build_search_result import (
34
- ECMWF_KEYWORDS,
35
- ECMWFSearch,
36
- keywords_to_mdt,
37
- )
33
+ from eodag.plugins.search.build_search_result import ECMWFSearch, ecmwf_mtd
38
34
  from eodag.utils import (
39
35
  DEFAULT_DOWNLOAD_TIMEOUT,
40
36
  DEFAULT_DOWNLOAD_WAIT,
@@ -94,7 +90,7 @@ class EcmwfApi(Api, ECMWFSearch):
94
90
  def __init__(self, provider: str, config: PluginConfig) -> None:
95
91
  # init self.config.metadata_mapping using Search Base plugin
96
92
  config.metadata_mapping = {
97
- **keywords_to_mdt(ECMWF_KEYWORDS, "ecmwf"),
93
+ **ecmwf_mtd(),
98
94
  **config.metadata_mapping,
99
95
  }
100
96
  Search.__init__(self, provider, config)
@@ -128,7 +128,7 @@ class UsgsApi(Api):
128
128
  api.logout()
129
129
  continue
130
130
  except USGSError as e:
131
- if i == 0:
131
+ if i == 0 and os.path.isfile(api.TMPFILE):
132
132
  # `.usgs` API file key might be obsolete
133
133
  # Remove it and try again
134
134
  os.remove(api.TMPFILE)
@@ -379,6 +379,12 @@ class OIDCAuthorizationCodeFlowAuth(OIDCRefreshTokenBase):
379
379
 
380
380
  login_document = etree.HTML(authorization_response.text)
381
381
  login_forms = login_document.xpath(self.config.login_form_xpath)
382
+
383
+ if not login_forms:
384
+ # we assume user is already logged in
385
+ # no form found because we got redirected to the redirect_uri
386
+ return authorization_response
387
+
382
388
  login_form = login_forms[0]
383
389
 
384
390
  # Get the form data to pass to the login form from config or from the login form
@@ -60,6 +60,7 @@ from eodag.utils.exceptions import (
60
60
  NotAvailableError,
61
61
  TimeOutError,
62
62
  )
63
+ from eodag.utils.s3 import open_s3_zipped_object
63
64
 
64
65
  if TYPE_CHECKING:
65
66
  from boto3.resources.collection import ResourceCollection
@@ -195,6 +196,7 @@ AWS_AUTH_ERROR_MESSAGES = [
195
196
  "AccessDenied",
196
197
  "InvalidAccessKeyId",
197
198
  "SignatureDoesNotMatch",
199
+ "InvalidRequest",
198
200
  ]
199
201
 
200
202
 
@@ -232,6 +234,7 @@ class AwsDownload(Download):
232
234
  super(AwsDownload, self).__init__(provider, config)
233
235
  self.requester_pays = getattr(self.config, "requester_pays", False)
234
236
  self.s3_session: Optional[boto3.session.Session] = None
237
+ self.s3_resource: Optional[boto3.resources.base.ServiceResource] = None
235
238
 
236
239
  def download(
237
240
  self,
@@ -289,10 +292,10 @@ class AwsDownload(Download):
289
292
  asset_filter = kwargs.get("asset", None)
290
293
  if asset_filter:
291
294
  build_safe = False
295
+ ignore_assets = False
292
296
  else:
293
297
  build_safe = product_conf.get("build_safe", False)
294
-
295
- ignore_assets = getattr(self.config, "ignore_assets", False)
298
+ ignore_assets = getattr(self.config, "ignore_assets", False)
296
299
 
297
300
  # product conf overrides provider conf for "flatten_top_dirs"
298
301
  flatten_top_dirs = product_conf.get(
@@ -325,19 +328,32 @@ class AwsDownload(Download):
325
328
  bucket_names_and_prefixes, auth
326
329
  )
327
330
 
331
+ # files in zip
332
+ updated_bucket_names_and_prefixes = self._download_file_in_zip(
333
+ product, bucket_names_and_prefixes, product_local_path, progress_callback
334
+ )
335
+ # prevent nothing-to-download errors if download was performed in zip
336
+ raise_error = (
337
+ False
338
+ if len(updated_bucket_names_and_prefixes) != len(bucket_names_and_prefixes)
339
+ else True
340
+ )
341
+
328
342
  # downloadable files
329
343
  unique_product_chunks = self._get_unique_products(
330
- bucket_names_and_prefixes,
344
+ updated_bucket_names_and_prefixes,
331
345
  authenticated_objects,
332
346
  asset_filter,
333
347
  ignore_assets,
334
348
  product,
349
+ raise_error=raise_error,
335
350
  )
336
351
 
337
352
  total_size = sum([p.size for p in unique_product_chunks]) or None
338
353
 
339
354
  # download
340
- progress_callback.reset(total=total_size)
355
+ if len(unique_product_chunks) > 0:
356
+ progress_callback.reset(total=total_size)
341
357
  try:
342
358
  for product_chunk in unique_product_chunks:
343
359
  try:
@@ -389,6 +405,52 @@ class AwsDownload(Download):
389
405
 
390
406
  return product_local_path
391
407
 
408
+ def _download_file_in_zip(
409
+ self, product, bucket_names_and_prefixes, product_local_path, progress_callback
410
+ ):
411
+ """
412
+ Download file in zip from a prefix like `foo/bar.zip!file.txt`
413
+ """
414
+ if self.s3_resource is None:
415
+ logger.debug("Cannot check files in s3 zip without s3 resource")
416
+ return bucket_names_and_prefixes
417
+
418
+ s3_client = self.s3_resource.meta.client
419
+
420
+ downloaded = []
421
+ for i, pack in enumerate(bucket_names_and_prefixes):
422
+ bucket_name, prefix = pack
423
+ if ".zip!" in prefix:
424
+ splitted_path = prefix.split(".zip!")
425
+ zip_prefix = f"{splitted_path[0]}.zip"
426
+ rel_path = splitted_path[-1]
427
+ dest_file = os.path.join(product_local_path, rel_path)
428
+ dest_abs_path_dir = os.path.dirname(dest_file)
429
+ if not os.path.isdir(dest_abs_path_dir):
430
+ os.makedirs(dest_abs_path_dir)
431
+
432
+ with open_s3_zipped_object(
433
+ bucket_name, zip_prefix, s3_client, partial=False
434
+ ) as zip_file:
435
+ # file size
436
+ file_info = zip_file.getinfo(rel_path)
437
+ progress_callback.reset(total=file_info.file_size)
438
+ with zip_file.open(rel_path) as extracted, open(
439
+ dest_file, "wb"
440
+ ) as output_file:
441
+ # Read in 1MB chunks
442
+ for zchunk in iter(lambda: extracted.read(1024 * 1024), b""):
443
+ output_file.write(zchunk)
444
+ progress_callback(len(zchunk))
445
+
446
+ downloaded.append(i)
447
+
448
+ return [
449
+ pack
450
+ for i, pack in enumerate(bucket_names_and_prefixes)
451
+ if i not in downloaded
452
+ ]
453
+
392
454
  def _download_preparation(
393
455
  self,
394
456
  product: EOProduct,
@@ -396,10 +458,12 @@ class AwsDownload(Download):
396
458
  **kwargs: Unpack[DownloadConf],
397
459
  ) -> tuple[Optional[str], Optional[str]]:
398
460
  """
399
- preparation for the download:
461
+ Preparation for the download:
462
+
400
463
  - check if file was already downloaded
401
464
  - get file path
402
465
  - create directories
466
+
403
467
  :param product: product to be downloaded
404
468
  :param progress_callback: progress callback to be used
405
469
  :param kwargs: additional arguments
@@ -423,7 +487,8 @@ class AwsDownload(Download):
423
487
 
424
488
  def _configure_safe_build(self, build_safe: bool, product: EOProduct):
425
489
  """
426
- updates the product properties with fetch metadata if safe build is enabled
490
+ Updates the product properties with fetch metadata if safe build is enabled
491
+
427
492
  :param build_safe: if safe build is enabled
428
493
  :param product: product to be updated
429
494
  """
@@ -513,10 +578,11 @@ class AwsDownload(Download):
513
578
  auth: Optional[Union[AuthBase, S3SessionKwargs]] = None,
514
579
  ) -> tuple[dict[str, Any], ResourceCollection]:
515
580
  """
516
- authenticates with s3 and retrieves the available objects
517
- raises an error when authentication is not possible
581
+ Authenticates with s3 and retrieves the available objects
582
+
518
583
  :param bucket_names_and_prefixes: list of bucket names and corresponding path prefixes
519
584
  :param auth: authentication information
585
+ :raises AuthenticationError: authentication is not possible
520
586
  :return: authenticated objects per bucket, list of available objects
521
587
  """
522
588
  if not isinstance(auth, (dict, type(None))):
@@ -583,14 +649,17 @@ class AwsDownload(Download):
583
649
  asset_filter: Optional[str],
584
650
  ignore_assets: bool,
585
651
  product: EOProduct,
652
+ raise_error: bool = True,
586
653
  ) -> set[Any]:
587
654
  """
588
- retrieve unique product chunks based on authenticated objects and asset filters
655
+ Retrieve unique product chunks based on authenticated objects and asset filters
656
+
589
657
  :param bucket_names_and_prefixes: list of bucket names and corresponding path prefixes
590
658
  :param authenticated_objects: available objects per bucket
591
659
  :param asset_filter: text for which assets should be filtered
592
660
  :param ignore_assets: if product instead of individual assets should be used
593
661
  :param product: product that shall be downloaded
662
+ :param raise_error: raise error if there is nothing to download
594
663
  :return: set of product chunks that can be downloaded
595
664
  """
596
665
  product_chunks: list[Any] = []
@@ -612,12 +681,12 @@ class AwsDownload(Download):
612
681
  unique_product_chunks,
613
682
  )
614
683
  )
615
- if not unique_product_chunks:
684
+ if not unique_product_chunks and raise_error:
616
685
  raise NotAvailableError(
617
686
  rf"No file basename matching re.fullmatch(r'{asset_filter}') was found in {product.remote_location}"
618
687
  )
619
688
 
620
- if not unique_product_chunks:
689
+ if not unique_product_chunks and raise_error:
621
690
  raise NoMatchingProductType("No product found to download.")
622
691
 
623
692
  return unique_product_chunks
@@ -701,6 +770,13 @@ class AwsDownload(Download):
701
770
  bucket_names_and_prefixes, auth
702
771
  )
703
772
 
773
+ # stream not implemented for prefixes like `foo/bar.zip!file.txt`
774
+ for _, prefix in bucket_names_and_prefixes:
775
+ if prefix and ".zip!" in prefix:
776
+ raise NotImplementedError(
777
+ "Download streaming is not implemented for files in zip on S3"
778
+ )
779
+
704
780
  # downloadable files
705
781
  unique_product_chunks = self._get_unique_products(
706
782
  bucket_names_and_prefixes,
@@ -935,6 +1011,7 @@ class AwsDownload(Download):
935
1011
  objects = s3_resource.Bucket(bucket_name).objects
936
1012
  list(objects.filter(Prefix=prefix).limit(1))
937
1013
  self.s3_session = s3_session
1014
+ self.s3_resource = s3_resource
938
1015
  return objects
939
1016
  else:
940
1017
  return None
@@ -965,6 +1042,7 @@ class AwsDownload(Download):
965
1042
  objects = s3_resource.Bucket(bucket_name).objects
966
1043
  list(objects.filter(Prefix=prefix).limit(1))
967
1044
  self.s3_session = s3_session
1045
+ self.s3_resource = s3_resource
968
1046
  return objects
969
1047
  else:
970
1048
  return None
@@ -986,6 +1064,7 @@ class AwsDownload(Download):
986
1064
  objects = s3_resource.Bucket(bucket_name).objects
987
1065
  list(objects.filter(Prefix=prefix).limit(1))
988
1066
  self.s3_session = s3_session
1067
+ self.s3_resource = s3_resource
989
1068
  return objects
990
1069
 
991
1070
  def get_product_bucket_name_and_prefix(
@@ -150,7 +150,7 @@ class Search(PluginTopic):
150
150
  )
151
151
 
152
152
  def get_product_type_def_params(
153
- self, product_type: str, **kwargs: Any
153
+ self, product_type: str, format_variables: Optional[dict[str, Any]] = None
154
154
  ) -> dict[str, Any]:
155
155
  """Get the provider product type definition parameters and specific settings
156
156
 
@@ -171,7 +171,8 @@ class Search(PluginTopic):
171
171
  return {
172
172
  k: v
173
173
  for k, v in format_dict_items(
174
- self.config.products[GENERIC_PRODUCT_TYPE], **kwargs
174
+ self.config.products[GENERIC_PRODUCT_TYPE],
175
+ **(format_variables or {}),
175
176
  ).items()
176
177
  if v
177
178
  }