eodag 3.10.1__py3-none-any.whl → 4.0.0a2__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (75) hide show
  1. eodag/__init__.py +6 -1
  2. eodag/api/collection.py +353 -0
  3. eodag/api/core.py +606 -641
  4. eodag/api/product/__init__.py +3 -3
  5. eodag/api/product/_product.py +74 -56
  6. eodag/api/product/drivers/__init__.py +4 -46
  7. eodag/api/product/drivers/base.py +0 -28
  8. eodag/api/product/metadata_mapping.py +178 -216
  9. eodag/api/search_result.py +156 -15
  10. eodag/cli.py +83 -403
  11. eodag/config.py +81 -51
  12. eodag/plugins/apis/base.py +2 -2
  13. eodag/plugins/apis/ecmwf.py +36 -25
  14. eodag/plugins/apis/usgs.py +55 -40
  15. eodag/plugins/authentication/base.py +1 -3
  16. eodag/plugins/crunch/filter_date.py +3 -3
  17. eodag/plugins/crunch/filter_latest_intersect.py +2 -2
  18. eodag/plugins/crunch/filter_latest_tpl_name.py +1 -1
  19. eodag/plugins/download/aws.py +46 -42
  20. eodag/plugins/download/base.py +13 -14
  21. eodag/plugins/download/http.py +65 -65
  22. eodag/plugins/manager.py +28 -29
  23. eodag/plugins/search/__init__.py +6 -4
  24. eodag/plugins/search/base.py +131 -80
  25. eodag/plugins/search/build_search_result.py +245 -173
  26. eodag/plugins/search/cop_marine.py +87 -56
  27. eodag/plugins/search/csw.py +47 -37
  28. eodag/plugins/search/qssearch.py +653 -429
  29. eodag/plugins/search/stac_list_assets.py +1 -1
  30. eodag/plugins/search/static_stac_search.py +43 -44
  31. eodag/resources/{product_types.yml → collections.yml} +2594 -2453
  32. eodag/resources/ext_collections.json +1 -1
  33. eodag/resources/ext_product_types.json +1 -1
  34. eodag/resources/providers.yml +2706 -2733
  35. eodag/resources/stac_provider.yml +50 -92
  36. eodag/resources/user_conf_template.yml +9 -0
  37. eodag/types/__init__.py +2 -0
  38. eodag/types/queryables.py +70 -91
  39. eodag/types/search_args.py +1 -1
  40. eodag/utils/__init__.py +97 -21
  41. eodag/utils/dates.py +0 -12
  42. eodag/utils/exceptions.py +6 -6
  43. eodag/utils/free_text_search.py +3 -3
  44. eodag/utils/repr.py +2 -0
  45. {eodag-3.10.1.dist-info → eodag-4.0.0a2.dist-info}/METADATA +13 -99
  46. eodag-4.0.0a2.dist-info/RECORD +93 -0
  47. {eodag-3.10.1.dist-info → eodag-4.0.0a2.dist-info}/entry_points.txt +0 -4
  48. eodag/plugins/authentication/oauth.py +0 -60
  49. eodag/plugins/download/creodias_s3.py +0 -71
  50. eodag/plugins/download/s3rest.py +0 -351
  51. eodag/plugins/search/data_request_search.py +0 -565
  52. eodag/resources/stac.yml +0 -294
  53. eodag/resources/stac_api.yml +0 -2105
  54. eodag/rest/__init__.py +0 -24
  55. eodag/rest/cache.py +0 -70
  56. eodag/rest/config.py +0 -67
  57. eodag/rest/constants.py +0 -26
  58. eodag/rest/core.py +0 -764
  59. eodag/rest/errors.py +0 -210
  60. eodag/rest/server.py +0 -604
  61. eodag/rest/server.wsgi +0 -6
  62. eodag/rest/stac.py +0 -1032
  63. eodag/rest/templates/README +0 -1
  64. eodag/rest/types/__init__.py +0 -18
  65. eodag/rest/types/collections_search.py +0 -44
  66. eodag/rest/types/eodag_search.py +0 -386
  67. eodag/rest/types/queryables.py +0 -174
  68. eodag/rest/types/stac_search.py +0 -272
  69. eodag/rest/utils/__init__.py +0 -207
  70. eodag/rest/utils/cql_evaluate.py +0 -119
  71. eodag/rest/utils/rfc3339.py +0 -64
  72. eodag-3.10.1.dist-info/RECORD +0 -116
  73. {eodag-3.10.1.dist-info → eodag-4.0.0a2.dist-info}/WHEEL +0 -0
  74. {eodag-3.10.1.dist-info → eodag-4.0.0a2.dist-info}/licenses/LICENSE +0 -0
  75. {eodag-3.10.1.dist-info → eodag-4.0.0a2.dist-info}/top_level.txt +0 -0
@@ -31,10 +31,10 @@ from usgs import USGSAuthExpiredError, USGSError, api
31
31
 
32
32
  from eodag.api.product import EOProduct
33
33
  from eodag.api.product.metadata_mapping import (
34
- DEFAULT_METADATA_MAPPING,
35
34
  mtd_cfg_as_conversion_and_querypath,
36
35
  properties_from_json,
37
36
  )
37
+ from eodag.api.search_result import SearchResult
38
38
  from eodag.plugins.apis.base import Api
39
39
  from eodag.plugins.search import PreparedSearch
40
40
  from eodag.utils import (
@@ -42,7 +42,7 @@ from eodag.utils import (
42
42
  DEFAULT_DOWNLOAD_WAIT,
43
43
  DEFAULT_ITEMS_PER_PAGE,
44
44
  DEFAULT_PAGE,
45
- GENERIC_PRODUCT_TYPE,
45
+ GENERIC_COLLECTION,
46
46
  USER_AGENT,
47
47
  ProgressCallback,
48
48
  format_dict_items,
@@ -50,7 +50,7 @@ from eodag.utils import (
50
50
  )
51
51
  from eodag.utils.exceptions import (
52
52
  AuthenticationError,
53
- NoMatchingProductType,
53
+ NoMatchingCollection,
54
54
  NotAvailableError,
55
55
  RequestError,
56
56
  ValidationError,
@@ -60,9 +60,7 @@ if TYPE_CHECKING:
60
60
  from mypy_boto3_s3 import S3ServiceResource
61
61
  from requests.auth import AuthBase
62
62
 
63
- from eodag.api.search_result import SearchResult
64
63
  from eodag.config import PluginConfig
65
- from eodag.types import S3SessionKwargs
66
64
  from eodag.types.download_args import DownloadConf
67
65
  from eodag.utils import DownloadedCallback, Unpack
68
66
 
@@ -101,7 +99,7 @@ class UsgsApi(Api):
101
99
  # Same method as in base.py, Search.__init__()
102
100
  # Prepare the metadata mapping
103
101
  # Do a shallow copy, the structure is flat enough for this to be sufficient
104
- metas: dict[str, Any] = DEFAULT_METADATA_MAPPING.copy()
102
+ metas: dict[str, Any] = {}
105
103
  # Update the defaults with the mapping value. This will add any new key
106
104
  # added by the provider mapping that is not in the default metadata.
107
105
  metas.update(self.config.metadata_mapping)
@@ -140,31 +138,38 @@ class UsgsApi(Api):
140
138
  self,
141
139
  prep: PreparedSearch = PreparedSearch(),
142
140
  **kwargs: Any,
143
- ) -> tuple[list[EOProduct], Optional[int]]:
141
+ ) -> SearchResult:
144
142
  """Search for data on USGS catalogues"""
145
- page = prep.page if prep.page is not None else DEFAULT_PAGE
143
+ token = (
144
+ int(prep.next_page_token)
145
+ if prep.next_page_token is not None
146
+ else DEFAULT_PAGE
147
+ )
146
148
  items_per_page = (
147
149
  prep.items_per_page
148
- if prep.items_per_page is not None
149
- else DEFAULT_ITEMS_PER_PAGE
150
+ or kwargs.pop("max_results", None)
151
+ or DEFAULT_ITEMS_PER_PAGE
150
152
  )
151
- product_type = kwargs.get("productType")
152
- if product_type is None:
153
- raise NoMatchingProductType(
154
- "Cannot search on USGS without productType specified"
153
+ search_params = {"items_per_page": items_per_page} | kwargs
154
+ collection = kwargs.get("collection")
155
+ if collection is None:
156
+ raise NoMatchingCollection(
157
+ "Cannot search on USGS without collection specified"
155
158
  )
156
159
  if kwargs.get("sort_by"):
157
160
  raise ValidationError("USGS does not support sorting feature")
158
161
 
159
162
  self.authenticate()
160
163
 
161
- product_type_def_params = self.config.products.get( # type: ignore
162
- product_type,
163
- self.config.products[GENERIC_PRODUCT_TYPE], # type: ignore
164
+ collection_def_params = self.config.products.get( # type: ignore
165
+ collection,
166
+ self.config.products[GENERIC_COLLECTION], # type: ignore
164
167
  )
165
- usgs_dataset = format_dict_items(product_type_def_params, **kwargs)["dataset"]
166
- start_date = kwargs.pop("startTimeFromAscendingNode", None)
167
- end_date = kwargs.pop("completionTimeFromAscendingNode", None)
168
+ usgs_collection = format_dict_items(collection_def_params, **kwargs)[
169
+ "_collection"
170
+ ]
171
+ start_date = kwargs.pop("start_datetime", None)
172
+ end_date = kwargs.pop("end_datetime", None)
168
173
  geom = kwargs.pop("geometry", None)
169
174
  footprint: dict[str, str] = {}
170
175
  if hasattr(geom, "bounds"):
@@ -196,19 +201,21 @@ class UsgsApi(Api):
196
201
  ll=lower_left,
197
202
  ur=upper_right,
198
203
  max_results=items_per_page,
199
- starting_number=(1 + (page - 1) * items_per_page),
204
+ starting_number=token,
200
205
  )
201
206
 
202
207
  # search by id
203
208
  if searched_id := kwargs.get("id"):
204
- dataset_filters = api.dataset_filters(usgs_dataset)
209
+ dataset_filters = api.dataset_filters(usgs_collection)
205
210
  # ip pattern set as parameter queryable (first element of param conf list)
206
211
  id_pattern = self.config.metadata_mapping["id"][0]
207
212
  # loop on matching dataset_filters until one returns expected results
208
213
  for dataset_filter in dataset_filters["data"]:
209
214
  if id_pattern in dataset_filter["searchSql"]:
210
215
  logger.debug(
211
- f"Try using {dataset_filter['searchSql']} dataset filter to search by id on {usgs_dataset}"
216
+ "Try using %s dataset filter to search by id on %s",
217
+ dataset_filter["searchSql"],
218
+ usgs_collection,
212
219
  )
213
220
  full_api_search_kwargs = {
214
221
  "where": {
@@ -218,19 +225,19 @@ class UsgsApi(Api):
218
225
  **api_search_kwargs,
219
226
  }
220
227
  logger.info(
221
- f"Sending search request for {usgs_dataset} with {full_api_search_kwargs}"
228
+ f"Sending search request for {usgs_collection} with {full_api_search_kwargs}"
222
229
  )
223
230
  results = api.scene_search(
224
- usgs_dataset, **full_api_search_kwargs
231
+ usgs_collection, **full_api_search_kwargs
225
232
  )
226
233
  if len(results["data"]["results"]) == 1:
227
234
  # search by id using this dataset_filter succeeded
228
235
  break
229
236
  else:
230
237
  logger.info(
231
- f"Sending search request for {usgs_dataset} with {api_search_kwargs}"
238
+ f"Sending search request for {usgs_collection} with {api_search_kwargs}"
232
239
  )
233
- results = api.scene_search(usgs_dataset, **api_search_kwargs)
240
+ results = api.scene_search(usgs_collection, **api_search_kwargs)
234
241
 
235
242
  # update results with storage info from download_options()
236
243
  results_by_entity_id = {
@@ -240,7 +247,7 @@ class UsgsApi(Api):
240
247
  f"Adapting {len(results_by_entity_id)} plugin results to eodag product representation"
241
248
  )
242
249
  download_options = api.download_options(
243
- usgs_dataset, list(results_by_entity_id.keys())
250
+ usgs_collection, list(results_by_entity_id.keys())
244
251
  )
245
252
  if download_options.get("data") is not None:
246
253
  for download_option in download_options["data"]:
@@ -262,7 +269,7 @@ class UsgsApi(Api):
262
269
  results["data"]["results"] = list(results_by_entity_id.values())
263
270
 
264
271
  for result in results["data"]["results"]:
265
- result["productType"] = usgs_dataset
272
+ result["collection"] = usgs_collection
266
273
 
267
274
  product_properties = properties_from_json(
268
275
  result, self.config.metadata_mapping
@@ -270,7 +277,7 @@ class UsgsApi(Api):
270
277
 
271
278
  final.append(
272
279
  EOProduct(
273
- productType=product_type,
280
+ collection=collection,
274
281
  provider=self.provider,
275
282
  properties=product_properties,
276
283
  geometry=footprint,
@@ -278,7 +285,7 @@ class UsgsApi(Api):
278
285
  )
279
286
  except USGSError as e:
280
287
  logger.warning(
281
- f"Product type {usgs_dataset} may not exist on USGS EE catalog"
288
+ f"Collection {usgs_collection} may not exist on USGS EE catalog"
282
289
  )
283
290
  api.logout()
284
291
  raise RequestError.from_error(e) from e
@@ -292,12 +299,18 @@ class UsgsApi(Api):
292
299
  else:
293
300
  total_results = 0
294
301
 
295
- return final, total_results
302
+ formated_result = SearchResult(
303
+ final,
304
+ total_results,
305
+ search_params=search_params,
306
+ next_page_token=results["data"]["nextRecord"],
307
+ )
308
+ return formated_result
296
309
 
297
310
  def download(
298
311
  self,
299
312
  product: EOProduct,
300
- auth: Optional[Union[AuthBase, S3SessionKwargs, S3ServiceResource]] = None,
313
+ auth: Optional[Union[AuthBase, S3ServiceResource]] = None,
301
314
  progress_callback: Optional[ProgressCallback] = None,
302
315
  wait: float = DEFAULT_DOWNLOAD_WAIT,
303
316
  timeout: float = DEFAULT_DOWNLOAD_TIMEOUT,
@@ -314,7 +327,7 @@ class UsgsApi(Api):
314
327
  output_extension = cast(
315
328
  str,
316
329
  self.config.products.get( # type: ignore
317
- product.product_type, self.config.products[GENERIC_PRODUCT_TYPE] # type: ignore
330
+ product.collection, self.config.products[GENERIC_COLLECTION] # type: ignore
318
331
  ).get("output_extension", ".tar.gz"),
319
332
  )
320
333
  kwargs["output_extension"] = kwargs.get("output_extension", output_extension)
@@ -335,11 +348,13 @@ class UsgsApi(Api):
335
348
  raise NotAvailableError(
336
349
  f"No USGS products found for {product.properties['id']}"
337
350
  )
338
-
351
+ usgs_dataset = self.config.products.get(product.collection, {}).get(
352
+ "_collection", GENERIC_COLLECTION
353
+ )
339
354
  download_request_results = api.download_request(
340
- product.properties["productType"],
341
- product.properties["entityId"],
342
- product.properties["productId"],
355
+ usgs_dataset,
356
+ product.properties["usgs:entityId"],
357
+ product.properties["usgs:productId"],
343
358
  )
344
359
 
345
360
  req_urls: list[str] = []
@@ -422,7 +437,7 @@ class UsgsApi(Api):
422
437
  download_request(product, fs_path, progress_callback, **kwargs)
423
438
 
424
439
  with open(record_filename, "w") as fh:
425
- fh.write(product.properties["downloadLink"])
440
+ fh.write(product.properties["eodag:download_link"])
426
441
  logger.debug(f"Download recorded in {record_filename}")
427
442
 
428
443
  api.logout()
@@ -466,7 +481,7 @@ class UsgsApi(Api):
466
481
  def download_all(
467
482
  self,
468
483
  products: SearchResult,
469
- auth: Optional[Union[AuthBase, S3SessionKwargs]] = None,
484
+ auth: Optional[Union[AuthBase, S3ServiceResource]] = None,
470
485
  downloaded_callback: Optional[DownloadedCallback] = None,
471
486
  progress_callback: Optional[ProgressCallback] = None,
472
487
  wait: float = DEFAULT_DOWNLOAD_WAIT,
@@ -27,8 +27,6 @@ if TYPE_CHECKING:
27
27
  from mypy_boto3_s3 import S3ServiceResource
28
28
  from requests.auth import AuthBase
29
29
 
30
- from eodag.types import S3SessionKwargs
31
-
32
30
 
33
31
  class Authentication(PluginTopic):
34
32
  """Plugins authentication Base plugin
@@ -42,7 +40,7 @@ class Authentication(PluginTopic):
42
40
  configuration that needs authentication and helps identifying it
43
41
  """
44
42
 
45
- def authenticate(self) -> Union[AuthBase, S3SessionKwargs, S3ServiceResource]:
43
+ def authenticate(self) -> Union[AuthBase, S3ServiceResource]:
46
44
  """Authenticate"""
47
45
  raise NotImplementedError
48
46
 
@@ -49,7 +49,7 @@ class FilterDate(Crunch):
49
49
  @staticmethod
50
50
  def sort_product_by_start_date(product: EOProduct) -> dt:
51
51
  """Get product start date"""
52
- start_date = product.properties.get("startTimeFromAscendingNode")
52
+ start_date = product.properties.get("start_datetime")
53
53
  if not start_date:
54
54
  # Retrieve year, month, day, hour, minute, second of EPOCH start
55
55
  epoch = time.gmtime(0)[:-3]
@@ -93,7 +93,7 @@ class FilterDate(Crunch):
93
93
  for product in products:
94
94
 
95
95
  # product start date
96
- product_start_str = product.properties.get("startTimeFromAscendingNode")
96
+ product_start_str = product.properties.get("start_datetime")
97
97
  if product_start_str:
98
98
  product_start = dateutil.parser.parse(product_start_str)
99
99
  if not product_start.tzinfo:
@@ -102,7 +102,7 @@ class FilterDate(Crunch):
102
102
  product_start = None
103
103
 
104
104
  # product end date
105
- product_end_str = product.properties.get("completionTimeFromAscendingNode")
105
+ product_end_str = product.properties.get("end_datetime")
106
106
  if product_end_str:
107
107
  product_end = dateutil.parser.parse(product_end_str)
108
108
  if not product_end.tzinfo:
@@ -46,7 +46,7 @@ class FilterLatestIntersect(Crunch):
46
46
  @staticmethod
47
47
  def sort_product_by_start_date(product: EOProduct) -> dt:
48
48
  """Get product start date"""
49
- start_date = product.properties.get("startTimeFromAscendingNode")
49
+ start_date = product.properties.get("start_datetime")
50
50
  if not start_date:
51
51
  # Retrieve year, month, day, hour, minute, second of EPOCH start
52
52
  epoch = time.gmtime(0)[:-3]
@@ -67,7 +67,7 @@ class FilterLatestIntersect(Crunch):
67
67
  logger.debug("Start filtering for latest products")
68
68
  if not products:
69
69
  return []
70
- # Warning: May crash if startTimeFromAscendingNode is not in the appropriate format
70
+ # Warning: May crash if start_datetime is not in the appropriate format
71
71
  products.sort(key=self.sort_product_by_start_date, reverse=True)
72
72
  filtered: list[EOProduct] = []
73
73
  search_extent: BaseGeometry
@@ -75,7 +75,7 @@ class FilterLatestByName(Crunch):
75
75
  logger.debug(
76
76
  "Latest product found for tileid=%s: date=%s",
77
77
  tileid,
78
- product.properties["startTimeFromAscendingNode"],
78
+ product.properties["start_datetime"],
79
79
  )
80
80
  filtered.append(product)
81
81
  processed.append(tileid)
@@ -44,6 +44,7 @@ from eodag.utils import (
44
44
  ProgressCallback,
45
45
  StreamResponse,
46
46
  flatten_top_directories,
47
+ format_string,
47
48
  get_bucket_name_and_prefix,
48
49
  path_to_uri,
49
50
  rename_subfolder,
@@ -53,7 +54,7 @@ from eodag.utils.exceptions import (
53
54
  AuthenticationError,
54
55
  DownloadError,
55
56
  MisconfiguredError,
56
- NoMatchingProductType,
57
+ NoMatchingCollection,
57
58
  NotAvailableError,
58
59
  TimeOutError,
59
60
  )
@@ -66,7 +67,6 @@ if TYPE_CHECKING:
66
67
  from eodag.api.product import EOProduct
67
68
  from eodag.api.search_result import SearchResult
68
69
  from eodag.config import PluginConfig
69
- from eodag.types import S3SessionKwargs
70
70
  from eodag.types.download_args import DownloadConf
71
71
  from eodag.utils import DownloadedCallback, Unpack
72
72
 
@@ -202,17 +202,17 @@ class AwsDownload(Download):
202
202
  * :attr:`~eodag.config.PluginConfig.flatten_top_dirs` (``bool``): if the directory structure
203
203
  should be flattened; default: ``True``
204
204
  * :attr:`~eodag.config.PluginConfig.ignore_assets` (``bool``): ignore assets and download
205
- using ``downloadLink``; default: ``False``
205
+ using ``eodag:download_link``; default: ``False``
206
206
  * :attr:`~eodag.config.PluginConfig.ssl_verify` (``bool``): if the ssl certificates should
207
207
  be verified in requests; default: ``True``
208
208
  * :attr:`~eodag.config.PluginConfig.bucket_path_level` (``int``): at which level of the
209
209
  path part of the url the bucket can be found; If no bucket_path_level is given, the bucket
210
210
  is taken from the first element of the netloc part.
211
- * :attr:`~eodag.config.PluginConfig.products` (``dict[str, dict[str, Any]``): product type
212
- specific config; the keys are the product types, the values are dictionaries which can contain the keys:
211
+ * :attr:`~eodag.config.PluginConfig.products` (``dict[str, dict[str, Any]``): collection
212
+ specific config; the keys are the collections, the values are dictionaries which can contain the keys:
213
213
 
214
- * **default_bucket** (``str``): bucket where the product type can be found
215
- * **complementary_url_key** (``str``): keys to add additional urls
214
+ * **default_bucket** (``str``): bucket where the collection can be found
215
+ * **complementary_url_key** (``str``): properties keys pointing to additional urls of content to download
216
216
  * **build_safe** (``bool``): if a SAFE (Standard Archive Format for Europe) product should
217
217
  be created; used for Sentinel products; default: False
218
218
  * **fetch_metadata** (``dict[str, Any]``): config for metadata to be fetched for the SAFE product
@@ -225,7 +225,7 @@ class AwsDownload(Download):
225
225
  def download(
226
226
  self,
227
227
  product: EOProduct,
228
- auth: Optional[Union[AuthBase, S3SessionKwargs, S3ServiceResource]] = None,
228
+ auth: Optional[Union[AuthBase, S3ServiceResource]] = None,
229
229
  progress_callback: Optional[ProgressCallback] = None,
230
230
  wait: float = DEFAULT_DOWNLOAD_WAIT,
231
231
  timeout: float = DEFAULT_DOWNLOAD_TIMEOUT,
@@ -234,7 +234,7 @@ class AwsDownload(Download):
234
234
  """Download method for AWS S3 API.
235
235
 
236
236
  The product can be downloaded as it is, or as SAFE-formatted product.
237
- SAFE-build is configured for a given provider and product type.
237
+ SAFE-build is configured for a given provider and collection.
238
238
  If the product title is configured to be updated during download and
239
239
  SAFE-formatted, its destination path will be:
240
240
  `{output_dir}/{title}`
@@ -266,9 +266,7 @@ class AwsDownload(Download):
266
266
  if not record_filename or not product_local_path:
267
267
  return product_local_path
268
268
 
269
- product_conf = getattr(self.config, "products", {}).get(
270
- product.product_type, {}
271
- )
269
+ product_conf = getattr(self.config, "products", {}).get(product.collection, {})
272
270
 
273
271
  # do not try to build SAFE if asset filter is used
274
272
  asset_filter = kwargs.get("asset")
@@ -368,7 +366,7 @@ class AwsDownload(Download):
368
366
  logger.warning("Unexpected error: %s" % e)
369
367
 
370
368
  # finalize safe product
371
- if build_safe and product.product_type and "S2_MSI" in product.product_type:
369
+ if build_safe and product.collection and "S2_MSI" in product.collection:
372
370
  self.finalize_s2_safe_product(product_local_path)
373
371
  # flatten directory structure
374
372
  elif flatten_top_dirs:
@@ -479,17 +477,15 @@ class AwsDownload(Download):
479
477
  :param build_safe: if safe build is enabled
480
478
  :param product: product to be updated
481
479
  """
482
- product_conf = getattr(self.config, "products", {}).get(
483
- product.product_type, {}
484
- )
480
+ product_conf = getattr(self.config, "products", {}).get(product.collection, {})
485
481
  ssl_verify = getattr(self.config, "ssl_verify", True)
486
482
  timeout = getattr(self.config, "timeout", HTTP_REQ_TIMEOUT)
487
483
 
488
484
  if build_safe and "fetch_metadata" in product_conf.keys():
489
485
  fetch_format = product_conf["fetch_metadata"]["fetch_format"]
490
486
  update_metadata = product_conf["fetch_metadata"]["update_metadata"]
491
- fetch_url = product_conf["fetch_metadata"]["fetch_url"].format(
492
- **product.properties
487
+ fetch_url = format_string(
488
+ None, product_conf["fetch_metadata"]["fetch_url"], **product.properties
493
489
  )
494
490
  logger.info("Fetching extra metadata from %s" % fetch_url)
495
491
  try:
@@ -619,14 +615,14 @@ class AwsDownload(Download):
619
615
  )
620
616
 
621
617
  if not unique_product_chunks and raise_error:
622
- raise NoMatchingProductType("No product found to download.")
618
+ raise NoMatchingCollection("No product found to download.")
623
619
 
624
620
  return unique_product_chunks
625
621
 
626
622
  def _stream_download_dict(
627
623
  self,
628
624
  product: EOProduct,
629
- auth: Optional[Union[AuthBase, S3SessionKwargs, S3ServiceResource]] = None,
625
+ auth: Optional[Union[AuthBase, S3ServiceResource]] = None,
630
626
  byte_range: tuple[Optional[int], Optional[int]] = (None, None),
631
627
  compress: Literal["zip", "raw", "auto"] = "auto",
632
628
  wait: float = DEFAULT_DOWNLOAD_WAIT,
@@ -664,7 +660,7 @@ class AwsDownload(Download):
664
660
 
665
661
  #### SAFE Archive Support:
666
662
 
667
- If the product type supports SAFE structure and no `asset_regex` is specified (i.e., full product download),
663
+ If the collection supports SAFE structure and no `asset_regex` is specified (i.e., full product download),
668
664
  the method attempts to reconstruct a valid SAFE archive layout in the streamed output.
669
665
 
670
666
  :param product: The EO product to download.
@@ -680,9 +676,7 @@ class AwsDownload(Download):
680
676
  """
681
677
  asset_regex = kwargs.get("asset")
682
678
 
683
- product_conf = getattr(self.config, "products", {}).get(
684
- product.product_type, {}
685
- )
679
+ product_conf = getattr(self.config, "products", {}).get(product.collection, {})
686
680
 
687
681
  build_safe = (
688
682
  False if asset_regex is not None else product_conf.get("build_safe", False)
@@ -716,7 +710,7 @@ class AwsDownload(Download):
716
710
  ignore_assets,
717
711
  product,
718
712
  )
719
- if auth and isinstance(auth, boto3.resource("s3").__class__):
713
+ if auth and isinstance(auth, boto3.resources.base.ServiceResource):
720
714
  s3_resource = auth
721
715
  else:
722
716
  s3_resource = boto3.resource(
@@ -724,9 +718,7 @@ class AwsDownload(Download):
724
718
  endpoint_url=getattr(self.config, "s3_endpoint", None),
725
719
  )
726
720
 
727
- product_conf = getattr(self.config, "products", {}).get(
728
- product.product_type, {}
729
- )
721
+ product_conf = getattr(self.config, "products", {}).get(product.collection, {})
730
722
  flatten_top_dirs = product_conf.get(
731
723
  "flatten_top_dirs", getattr(self.config, "flatten_top_dirs", True)
732
724
  )
@@ -814,7 +806,7 @@ class AwsDownload(Download):
814
806
  if bucket is None:
815
807
  bucket = (
816
808
  getattr(self.config, "products", {})
817
- .get(product.product_type, {})
809
+ .get(product.collection, {})
818
810
  .get("default_bucket", "")
819
811
  )
820
812
 
@@ -925,7 +917,7 @@ class AwsDownload(Download):
925
917
  s2_processing_level: str = ""
926
918
  s1_title_suffix: Optional[str] = None
927
919
  # S2 common
928
- if product.product_type and "S2_MSI" in product.product_type:
920
+ if product.collection and "S2_MSI" in product.collection:
929
921
  title_search: Optional[re.Match[str]] = re.search(
930
922
  r"^\w+_\w+_(\w+)_(\w+)_(\w+)_(\w+)_(\w+)$",
931
923
  product.properties["title"],
@@ -937,9 +929,9 @@ class AwsDownload(Download):
937
929
  product.properties.get("originalSceneID", ""),
938
930
  )
939
931
  ds_dir = ds_dir_search.group(1) if ds_dir_search else 0
940
- s2_processing_level = product.product_type.split("_")[-1]
932
+ s2_processing_level = product.collection.split("_")[-1]
941
933
  # S1 common
942
- elif product.product_type == "S1_SAR_GRD":
934
+ elif product.collection == "S1_SAR_GRD":
943
935
  s1_title_suffix_search = re.search(
944
936
  r"^.+_([A-Z0-9_]+_[A-Z0-9_]+_[A-Z0-9_]+_[A-Z0-9_]+)_\w+$",
945
937
  product.properties["title"],
@@ -950,6 +942,12 @@ class AwsDownload(Download):
950
942
  else None
951
943
  )
952
944
 
945
+ # S1 polarization mode
946
+ if product_title := product.properties.get("title"):
947
+ polarization_mode = re.sub(r".{14}([A-Z]{2}).*", r"\1", product_title)
948
+ else:
949
+ polarization_mode = None
950
+
953
951
  # S2 L2A Tile files -----------------------------------------------
954
952
  if matched := S2L2A_TILE_IMG_REGEX.match(chunk.key):
955
953
  found_dict = matched.groupdict()
@@ -1057,37 +1055,43 @@ class AwsDownload(Download):
1057
1055
  found_dict = matched.groupdict()
1058
1056
  product_path = "%s" % found_dict["file"]
1059
1057
  # S1 --------------------------------------------------------------
1060
- elif matched := S1_CALIB_REGEX.match(chunk.key):
1058
+ elif (
1059
+ matched := S1_CALIB_REGEX.match(chunk.key)
1060
+ ) and polarization_mode is not None:
1061
1061
  found_dict = matched.groupdict()
1062
1062
  product_path = "annotation/calibration/%s-%s-%s-grd-%s-%s-%03d.xml" % (
1063
1063
  found_dict["file_prefix"],
1064
- product.properties["platformSerialIdentifier"].lower(),
1064
+ product.properties["platform"].lower(),
1065
1065
  found_dict["file_beam"],
1066
1066
  found_dict["file_pol"],
1067
1067
  s1_title_suffix,
1068
- S1_IMG_NB_PER_POLAR.get(product.properties["polarizationMode"], {}).get(
1068
+ S1_IMG_NB_PER_POLAR.get(polarization_mode, {}).get(
1069
1069
  found_dict["file_pol"].upper(), 1
1070
1070
  ),
1071
1071
  )
1072
- elif matched := S1_ANNOT_REGEX.match(chunk.key):
1072
+ elif (
1073
+ matched := S1_ANNOT_REGEX.match(chunk.key)
1074
+ ) and polarization_mode is not None:
1073
1075
  found_dict = matched.groupdict()
1074
1076
  product_path = "annotation/%s-%s-grd-%s-%s-%03d.xml" % (
1075
- product.properties["platformSerialIdentifier"].lower(),
1077
+ product.properties["platform"].lower(),
1076
1078
  found_dict["file_beam"],
1077
1079
  found_dict["file_pol"],
1078
1080
  s1_title_suffix,
1079
- S1_IMG_NB_PER_POLAR.get(product.properties["polarizationMode"], {}).get(
1081
+ S1_IMG_NB_PER_POLAR.get(polarization_mode, {}).get(
1080
1082
  found_dict["file_pol"].upper(), 1
1081
1083
  ),
1082
1084
  )
1083
- elif matched := S1_MEAS_REGEX.match(chunk.key):
1085
+ elif (
1086
+ matched := S1_MEAS_REGEX.match(chunk.key)
1087
+ ) and polarization_mode is not None:
1084
1088
  found_dict = matched.groupdict()
1085
1089
  product_path = "measurement/%s-%s-grd-%s-%s-%03d.%s" % (
1086
- product.properties["platformSerialIdentifier"].lower(),
1090
+ product.properties["platform"].lower(),
1087
1091
  found_dict["file_beam"],
1088
1092
  found_dict["file_pol"],
1089
1093
  s1_title_suffix,
1090
- S1_IMG_NB_PER_POLAR.get(product.properties["polarizationMode"], {}).get(
1094
+ S1_IMG_NB_PER_POLAR.get(polarization_mode, {}).get(
1091
1095
  found_dict["file_pol"].upper(), 1
1092
1096
  ),
1093
1097
  found_dict["file_ext"],
@@ -1112,7 +1116,7 @@ class AwsDownload(Download):
1112
1116
  def download_all(
1113
1117
  self,
1114
1118
  products: SearchResult,
1115
- auth: Optional[Union[AuthBase, S3SessionKwargs]] = None,
1119
+ auth: Optional[Union[AuthBase, S3ServiceResource]] = None,
1116
1120
  downloaded_callback: Optional[DownloadedCallback] = None,
1117
1121
  progress_callback: Optional[ProgressCallback] = None,
1118
1122
  wait: float = DEFAULT_DOWNLOAD_WAIT,