eodag 3.1.0b1__py3-none-any.whl → 3.1.0b2__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (85) hide show
  1. eodag/api/core.py +59 -52
  2. eodag/api/product/_assets.py +5 -5
  3. eodag/api/product/_product.py +27 -12
  4. eodag/api/product/drivers/__init__.py +81 -4
  5. eodag/api/product/drivers/base.py +65 -4
  6. eodag/api/product/drivers/generic.py +65 -0
  7. eodag/api/product/drivers/sentinel1.py +97 -0
  8. eodag/api/product/drivers/sentinel2.py +95 -0
  9. eodag/api/product/metadata_mapping.py +62 -74
  10. eodag/api/search_result.py +13 -23
  11. eodag/cli.py +4 -4
  12. eodag/config.py +66 -69
  13. eodag/plugins/apis/base.py +1 -1
  14. eodag/plugins/apis/ecmwf.py +10 -9
  15. eodag/plugins/apis/usgs.py +11 -10
  16. eodag/plugins/authentication/aws_auth.py +16 -13
  17. eodag/plugins/authentication/base.py +5 -3
  18. eodag/plugins/authentication/header.py +3 -3
  19. eodag/plugins/authentication/keycloak.py +4 -4
  20. eodag/plugins/authentication/oauth.py +7 -3
  21. eodag/plugins/authentication/openid_connect.py +14 -14
  22. eodag/plugins/authentication/sas_auth.py +4 -4
  23. eodag/plugins/authentication/token.py +7 -7
  24. eodag/plugins/authentication/token_exchange.py +1 -1
  25. eodag/plugins/base.py +4 -4
  26. eodag/plugins/crunch/base.py +4 -4
  27. eodag/plugins/crunch/filter_date.py +4 -4
  28. eodag/plugins/crunch/filter_latest_intersect.py +6 -6
  29. eodag/plugins/crunch/filter_latest_tpl_name.py +7 -7
  30. eodag/plugins/crunch/filter_overlap.py +4 -4
  31. eodag/plugins/crunch/filter_property.py +4 -4
  32. eodag/plugins/download/aws.py +47 -66
  33. eodag/plugins/download/base.py +8 -17
  34. eodag/plugins/download/creodias_s3.py +2 -2
  35. eodag/plugins/download/http.py +30 -32
  36. eodag/plugins/download/s3rest.py +5 -4
  37. eodag/plugins/manager.py +10 -20
  38. eodag/plugins/search/__init__.py +6 -5
  39. eodag/plugins/search/base.py +35 -40
  40. eodag/plugins/search/build_search_result.py +69 -68
  41. eodag/plugins/search/cop_marine.py +22 -12
  42. eodag/plugins/search/creodias_s3.py +8 -78
  43. eodag/plugins/search/csw.py +11 -11
  44. eodag/plugins/search/data_request_search.py +16 -15
  45. eodag/plugins/search/qssearch.py +56 -52
  46. eodag/plugins/search/stac_list_assets.py +85 -0
  47. eodag/plugins/search/static_stac_search.py +3 -3
  48. eodag/resources/ext_product_types.json +1 -1
  49. eodag/resources/product_types.yml +288 -288
  50. eodag/resources/providers.yml +146 -6
  51. eodag/resources/stac_api.yml +2 -2
  52. eodag/resources/user_conf_template.yml +11 -0
  53. eodag/rest/cache.py +2 -2
  54. eodag/rest/config.py +3 -3
  55. eodag/rest/core.py +24 -24
  56. eodag/rest/errors.py +5 -5
  57. eodag/rest/server.py +3 -11
  58. eodag/rest/stac.py +40 -38
  59. eodag/rest/types/collections_search.py +3 -3
  60. eodag/rest/types/eodag_search.py +23 -23
  61. eodag/rest/types/queryables.py +13 -13
  62. eodag/rest/types/stac_search.py +15 -25
  63. eodag/rest/utils/__init__.py +11 -21
  64. eodag/rest/utils/cql_evaluate.py +6 -6
  65. eodag/rest/utils/rfc3339.py +2 -2
  66. eodag/types/__init__.py +24 -18
  67. eodag/types/bbox.py +2 -2
  68. eodag/types/download_args.py +2 -2
  69. eodag/types/queryables.py +5 -2
  70. eodag/types/search_args.py +4 -4
  71. eodag/types/whoosh.py +1 -3
  72. eodag/utils/__init__.py +81 -40
  73. eodag/utils/exceptions.py +2 -2
  74. eodag/utils/import_system.py +2 -2
  75. eodag/utils/requests.py +2 -2
  76. eodag/utils/rest.py +2 -2
  77. eodag/utils/s3.py +208 -0
  78. eodag/utils/stac_reader.py +10 -10
  79. {eodag-3.1.0b1.dist-info → eodag-3.1.0b2.dist-info}/METADATA +5 -4
  80. eodag-3.1.0b2.dist-info/RECORD +113 -0
  81. {eodag-3.1.0b1.dist-info → eodag-3.1.0b2.dist-info}/entry_points.txt +1 -0
  82. eodag-3.1.0b1.dist-info/RECORD +0 -108
  83. {eodag-3.1.0b1.dist-info → eodag-3.1.0b2.dist-info}/LICENSE +0 -0
  84. {eodag-3.1.0b1.dist-info → eodag-3.1.0b2.dist-info}/WHEEL +0 -0
  85. {eodag-3.1.0b1.dist-info → eodag-3.1.0b2.dist-info}/top_level.txt +0 -0
@@ -23,18 +23,7 @@ import logging
23
23
  import re
24
24
  from collections import OrderedDict
25
25
  from datetime import datetime, timedelta
26
- from typing import (
27
- TYPE_CHECKING,
28
- Annotated,
29
- Any,
30
- Dict,
31
- List,
32
- Optional,
33
- Set,
34
- Tuple,
35
- Union,
36
- cast,
37
- )
26
+ from typing import TYPE_CHECKING, Annotated, Any, Optional, Union, cast
38
27
  from urllib.parse import quote_plus, unquote_plus
39
28
 
40
29
  import geojson
@@ -61,9 +50,9 @@ from eodag.api.search_result import RawSearchResult
61
50
  from eodag.plugins.search import PreparedSearch
62
51
  from eodag.plugins.search.qssearch import PostJsonSearch, QueryStringSearch
63
52
  from eodag.types import json_field_definition_to_python
64
- from eodag.types.queryables import Queryables
53
+ from eodag.types.queryables import Queryables, QueryablesDict
65
54
  from eodag.utils import (
66
- HTTP_REQ_TIMEOUT,
55
+ DEFAULT_SEARCH_TIMEOUT,
67
56
  deepcopy,
68
57
  dict_items_recursive_sort,
69
58
  get_geometry_from_various,
@@ -205,8 +194,8 @@ COP_DS_KEYWORDS = [
205
194
 
206
195
 
207
196
  def keywords_to_mdt(
208
- keywords: List[str], prefix: Optional[str] = None
209
- ) -> Dict[str, Any]:
197
+ keywords: list[str], prefix: Optional[str] = None
198
+ ) -> dict[str, Any]:
210
199
  """
211
200
  Make metadata mapping dict from a list of keywords
212
201
 
@@ -223,7 +212,7 @@ def keywords_to_mdt(
223
212
  :param prefix: prefix to be added to the parameter in the mapping
224
213
  :return: metadata mapping dict
225
214
  """
226
- mdt: Dict[str, Any] = {}
215
+ mdt: dict[str, Any] = {}
227
216
  for keyword in keywords:
228
217
  key = f"{prefix}:{keyword}" if prefix else keyword
229
218
  mdt[key] = [keyword, f'$."{key}"']
@@ -237,6 +226,8 @@ def strip_quotes(value: Any) -> Any:
237
226
  'abc'
238
227
  >>> strip_quotes(["'abc'", '"def'])
239
228
  ['abc', 'def']
229
+ >>> strip_quotes({"'abc'": 'def"'})
230
+ {'abc': 'def'}
240
231
 
241
232
  :param value: value from which quotes should be removed (should be either str or list)
242
233
  :return: value without quotes
@@ -245,13 +236,13 @@ def strip_quotes(value: Any) -> Any:
245
236
  if isinstance(value, (list, tuple)):
246
237
  return [strip_quotes(v) for v in value]
247
238
  elif isinstance(value, dict):
248
- raise NotImplementedError("Dict value is not supported.")
239
+ return {strip_quotes(k): strip_quotes(v) for k, v in value.items()}
249
240
  else:
250
241
  return str(value).strip("'\"")
251
242
 
252
243
 
253
244
  def _update_properties_from_element(
254
- prop: Dict[str, Any], element: Dict[str, Any], values: List[str]
245
+ prop: dict[str, Any], element: dict[str, Any], values: list[str]
255
246
  ) -> None:
256
247
  """updates a property dict with the given values based on the information from the element dict
257
248
  e.g. the type is set based on the type of the element
@@ -333,7 +324,7 @@ class ECMWFSearch(PostJsonSearch):
333
324
  :param provider: An eodag providers configuration dictionary
334
325
  :param config: Search plugin configuration:
335
326
 
336
- * :attr:`~eodag.config.PluginConfig.remove_from_query` (``List[str]``): List of parameters
327
+ * :attr:`~eodag.config.PluginConfig.remove_from_query` (``list[str]``): List of parameters
337
328
  used to parse metadata but that must not be included to the query
338
329
  * :attr:`~eodag.config.PluginConfig.end_date_excluded` (``bool``): Set to `False` if
339
330
  provider does not include end date to search
@@ -350,9 +341,6 @@ class ECMWFSearch(PostJsonSearch):
350
341
  """
351
342
 
352
343
  def __init__(self, provider: str, config: PluginConfig) -> None:
353
- # cache fetching method
354
- self.fetch_data = functools.lru_cache()(self._fetch_data)
355
-
356
344
  config.metadata_mapping = {
357
345
  **keywords_to_mdt(ECMWF_KEYWORDS + COP_DS_KEYWORDS, "ecmwf"),
358
346
  **config.metadata_mapping,
@@ -402,7 +390,7 @@ class ECMWFSearch(PostJsonSearch):
402
390
  "metadata_mapping"
403
391
  ] = product_type_metadata_mapping
404
392
 
405
- def do_search(self, *args: Any, **kwargs: Any) -> List[Dict[str, Any]]:
393
+ def do_search(self, *args: Any, **kwargs: Any) -> list[dict[str, Any]]:
406
394
  """Should perform the actual search request.
407
395
 
408
396
  :param args: arguments to be used in the search
@@ -416,7 +404,7 @@ class ECMWFSearch(PostJsonSearch):
416
404
  self,
417
405
  prep: PreparedSearch = PreparedSearch(),
418
406
  **kwargs: Any,
419
- ) -> Tuple[List[EOProduct], Optional[int]]:
407
+ ) -> tuple[list[EOProduct], Optional[int]]:
420
408
  """Build ready-to-download SearchResult
421
409
 
422
410
  :param prep: :class:`~eodag.plugins.search.PreparedSearch` object containing information needed for the search
@@ -439,7 +427,7 @@ class ECMWFSearch(PostJsonSearch):
439
427
 
440
428
  def build_query_string(
441
429
  self, product_type: str, **kwargs: Any
442
- ) -> Tuple[Dict[str, Any], str]:
430
+ ) -> tuple[dict[str, Any], str]:
443
431
  """Build The query string using the search parameters
444
432
 
445
433
  :param product_type: product type id
@@ -464,7 +452,7 @@ class ECMWFSearch(PostJsonSearch):
464
452
  )
465
453
 
466
454
  def _preprocess_search_params(
467
- self, params: Dict[str, Any], product_type: Optional[str]
455
+ self, params: dict[str, Any], product_type: Optional[str]
468
456
  ) -> None:
469
457
  """Preprocess search parameters before making a request to the CDS API.
470
458
 
@@ -543,9 +531,23 @@ class ECMWFSearch(PostJsonSearch):
543
531
  if "geometry" in params:
544
532
  params["geometry"] = get_geometry_from_various(geometry=params["geometry"])
545
533
 
534
+ def _get_product_type_queryables(
535
+ self, product_type: Optional[str], alias: Optional[str], filters: dict[str, Any]
536
+ ) -> QueryablesDict:
537
+ """Override to set additional_properties to false."""
538
+ default_values: dict[str, Any] = deepcopy(
539
+ getattr(self.config, "products", {}).get(product_type, {})
540
+ )
541
+ default_values.pop("metadata_mapping", None)
542
+
543
+ filters["productType"] = product_type
544
+ queryables = self.discover_queryables(**{**default_values, **filters}) or {}
545
+
546
+ return QueryablesDict(additional_properties=False, **queryables)
547
+
546
548
  def discover_queryables(
547
549
  self, **kwargs: Any
548
- ) -> Optional[Dict[str, Annotated[Any, FieldInfo]]]:
550
+ ) -> Optional[dict[str, Annotated[Any, FieldInfo]]]:
549
551
  """Fetch queryables list from provider using its constraints file
550
552
 
551
553
  :param kwargs: additional filters for queryables (`productType` and other search
@@ -571,13 +573,13 @@ class ECMWFSearch(PostJsonSearch):
571
573
  getattr(self.config, "discover_queryables", {}).get("constraints_url", ""),
572
574
  **kwargs,
573
575
  )
574
- constraints: List[Dict[str, Any]] = self.fetch_data(constraints_url)
576
+ constraints: list[dict[str, Any]] = self._fetch_data(constraints_url)
575
577
 
576
578
  form_url = format_metadata(
577
579
  getattr(self.config, "discover_queryables", {}).get("form_url", ""),
578
580
  **kwargs,
579
581
  )
580
- form = self.fetch_data(form_url)
582
+ form: list[dict[str, Any]] = self._fetch_data(form_url)
581
583
 
582
584
  formated_kwargs = self.format_as_provider_keyword(
583
585
  product_type, processed_kwargs
@@ -600,18 +602,18 @@ class ECMWFSearch(PostJsonSearch):
600
602
  # we use non empty kwargs as default to integrate user inputs
601
603
  # it is needed because pydantic json schema does not represent "value"
602
604
  # but only "default"
603
- non_empty_formated: Dict[str, Any] = {
605
+ non_empty_formated: dict[str, Any] = {
604
606
  k: v
605
607
  for k, v in formated_kwargs.items()
606
608
  if v and (not isinstance(v, list) or all(v))
607
609
  }
608
- non_empty_kwargs: Dict[str, Any] = {
610
+ non_empty_kwargs: dict[str, Any] = {
609
611
  k: v
610
612
  for k, v in processed_kwargs.items()
611
613
  if v and (not isinstance(v, list) or all(v))
612
614
  }
613
615
 
614
- required_keywords: Set[str] = set()
616
+ required_keywords: set[str] = set()
615
617
 
616
618
  # calculate available values
617
619
  if constraints:
@@ -634,7 +636,7 @@ class ECMWFSearch(PostJsonSearch):
634
636
  return self.queryables_from_metadata_mapping(product_type)
635
637
  if "{" in values_url:
636
638
  values_url = values_url.format(productType=provider_product_type)
637
- data = self.fetch_data(values_url)
639
+ data = self._fetch_data(values_url)
638
640
  available_values = data["constraints"]
639
641
  required_keywords = data.get("required", [])
640
642
 
@@ -651,7 +653,9 @@ class ECMWFSearch(PostJsonSearch):
651
653
  "completionTimeFromAscendingNode",
652
654
  "geom",
653
655
  }
654
- and keyword.replace("ecmwf:", "") not in available_values
656
+ and keyword not in [f["name"] for f in form]
657
+ and keyword.replace("ecmwf:", "")
658
+ not in set(list(available_values.keys()) + [f["name"] for f in form])
655
659
  ):
656
660
  raise ValidationError(f"{keyword} is not a queryable parameter")
657
661
 
@@ -689,7 +693,7 @@ class ECMWFSearch(PostJsonSearch):
689
693
  # area is geom in EODAG.
690
694
  if queryables.pop("area", None):
691
695
  queryables["geom"] = Annotated[
692
- Union[str, Dict[str, float], BaseGeometry],
696
+ Union[str, dict[str, float], BaseGeometry],
693
697
  Field(
694
698
  None,
695
699
  description="Read EODAG documentation for all supported geometry format.",
@@ -700,10 +704,10 @@ class ECMWFSearch(PostJsonSearch):
700
704
 
701
705
  def available_values_from_constraints(
702
706
  self,
703
- constraints: list[Dict[str, Any]],
704
- input_keywords: Dict[str, Any],
705
- form_keywords: List[str],
706
- ) -> Dict[str, List[str]]:
707
+ constraints: list[dict[str, Any]],
708
+ input_keywords: dict[str, Any],
709
+ form_keywords: list[str],
710
+ ) -> dict[str, list[str]]:
707
711
  """
708
712
  Filter constraints using input_keywords. Return list of available queryables.
709
713
  All constraint entries must have the same parameters.
@@ -727,9 +731,9 @@ class ECMWFSearch(PostJsonSearch):
727
731
  )
728
732
 
729
733
  # filter constraint entries matching input keyword values
730
- filtered_constraints: List[Dict[str, Any]]
734
+ filtered_constraints: list[dict[str, Any]]
731
735
 
732
- parsed_keywords: List[str] = []
736
+ parsed_keywords: list[str] = []
733
737
  for keyword in ordered_keywords:
734
738
  values = input_keywords.get(keyword)
735
739
 
@@ -808,7 +812,7 @@ class ECMWFSearch(PostJsonSearch):
808
812
  parsed_keywords.append(keyword)
809
813
  constraints = filtered_constraints
810
814
 
811
- available_values: Dict[str, Any] = {k: set() for k in ordered_keywords}
815
+ available_values: dict[str, Any] = {k: set() for k in ordered_keywords}
812
816
 
813
817
  # we aggregate the constraint entries left
814
818
  for entry in constraints:
@@ -819,10 +823,10 @@ class ECMWFSearch(PostJsonSearch):
819
823
 
820
824
  def queryables_by_form(
821
825
  self,
822
- form: List[Dict[str, Any]],
823
- available_values: Dict[str, List[str]],
824
- defaults: Dict[str, Any],
825
- ) -> Dict[str, Annotated[Any, FieldInfo]]:
826
+ form: list[dict[str, Any]],
827
+ available_values: dict[str, list[str]],
828
+ defaults: dict[str, Any],
829
+ ) -> dict[str, Annotated[Any, FieldInfo]]:
826
830
  """
827
831
  Generate Annotated field definitions from form entries and available values
828
832
  Used by Copernicus services like cop_cds, cop_ads, cop_ewds.
@@ -832,9 +836,9 @@ class ECMWFSearch(PostJsonSearch):
832
836
  :param defaults: default values for the parameters
833
837
  :return: dict of annotated queryables
834
838
  """
835
- queryables: Dict[str, Annotated[Any, FieldInfo]] = {}
839
+ queryables: dict[str, Annotated[Any, FieldInfo]] = {}
836
840
 
837
- required_list: List[str] = []
841
+ required_list: list[str] = []
838
842
  for element in form:
839
843
  name: str = element["name"]
840
844
 
@@ -869,9 +873,6 @@ class ECMWFSearch(PostJsonSearch):
869
873
  if fields and (comment := fields[0].get("comment")):
870
874
  prop["description"] = comment
871
875
 
872
- if d := details.get("default"):
873
- default = default or (d[0] if fields else d)
874
-
875
876
  if name == "area" and isinstance(default, dict):
876
877
  default = list(default.values())
877
878
 
@@ -901,10 +902,10 @@ class ECMWFSearch(PostJsonSearch):
901
902
 
902
903
  def queryables_by_values(
903
904
  self,
904
- available_values: Dict[str, List[str]],
905
- required_keywords: List[str],
906
- defaults: Dict[str, Any],
907
- ) -> Dict[str, Annotated[Any, FieldInfo]]:
905
+ available_values: dict[str, list[str]],
906
+ required_keywords: list[str],
907
+ defaults: dict[str, Any],
908
+ ) -> dict[str, Annotated[Any, FieldInfo]]:
908
909
  """
909
910
  Generate Annotated field definitions from available values.
910
911
  Used by ECMWF data providers like dedt_lumi.
@@ -918,7 +919,7 @@ class ECMWFSearch(PostJsonSearch):
918
919
  # Needed to map constraints like "xxxx" to eodag parameter "ecmwf:xxxx"
919
920
  required = [ecmwf_format(k) for k in required_keywords]
920
921
 
921
- queryables: Dict[str, Annotated[Any, FieldInfo]] = {}
922
+ queryables: dict[str, Annotated[Any, FieldInfo]] = {}
922
923
  for name, values in available_values.items():
923
924
  # Rename keywords from form with metadata mapping.
924
925
  # Needed to map constraints like "xxxx" to eodag parameter "ecmwf:xxxx"
@@ -939,8 +940,8 @@ class ECMWFSearch(PostJsonSearch):
939
940
  return queryables
940
941
 
941
942
  def format_as_provider_keyword(
942
- self, product_type: str, properties: Dict[str, Any]
943
- ) -> Dict[str, Any]:
943
+ self, product_type: str, properties: dict[str, Any]
944
+ ) -> dict[str, Any]:
944
945
  """Return provider equivalent keyword names from EODAG keywords.
945
946
 
946
947
  :param product_type: product type id
@@ -973,12 +974,12 @@ class ECMWFSearch(PostJsonSearch):
973
974
  if hasattr(self, "auth") and isinstance(self.auth, AuthBase)
974
975
  else None
975
976
  )
976
- timeout = getattr(self.config, "timeout", HTTP_REQ_TIMEOUT)
977
- return fetch_json(url, auth=auth, timeout=timeout)
977
+ timeout = getattr(self.config, "timeout", DEFAULT_SEARCH_TIMEOUT)
978
+ return functools.lru_cache()(fetch_json)(url, auth=auth, timeout=timeout)
978
979
 
979
980
  def normalize_results(
980
981
  self, results: RawSearchResult, **kwargs: Any
981
- ) -> List[EOProduct]:
982
+ ) -> list[EOProduct]:
982
983
  """Build :class:`~eodag.api.product._product.EOProduct` from provider result
983
984
 
984
985
  :param results: Raw provider result as single dict in list
@@ -1150,7 +1151,7 @@ class MeteoblueSearch(ECMWFSearch):
1150
1151
  self,
1151
1152
  prep: PreparedSearch = PreparedSearch(),
1152
1153
  **kwargs: Any,
1153
- ) -> Tuple[List[str], int]:
1154
+ ) -> tuple[list[str], int]:
1154
1155
  """Wraps PostJsonSearch.collect_search_urls to force product count to 1
1155
1156
 
1156
1157
  :param prep: :class:`~eodag.plugins.search.PreparedSearch` object containing information for the search
@@ -1162,7 +1163,7 @@ class MeteoblueSearch(ECMWFSearch):
1162
1163
 
1163
1164
  def do_search(
1164
1165
  self, prep: PreparedSearch = PreparedSearch(items_per_page=None), **kwargs: Any
1165
- ) -> List[Dict[str, Any]]:
1166
+ ) -> list[dict[str, Any]]:
1166
1167
  """Perform the actual search request, and return result in a single element.
1167
1168
 
1168
1169
  :param prep: :class:`~eodag.plugins.search.PreparedSearch` object containing information for the search
@@ -1182,7 +1183,7 @@ class MeteoblueSearch(ECMWFSearch):
1182
1183
 
1183
1184
  def build_query_string(
1184
1185
  self, product_type: str, **kwargs: Any
1185
- ) -> Tuple[Dict[str, Any], str]:
1186
+ ) -> tuple[dict[str, Any], str]:
1186
1187
  """Build The query string using the search parameters
1187
1188
 
1188
1189
  :param product_type: product type id
@@ -1221,7 +1222,7 @@ class WekeoECMWFSearch(ECMWFSearch):
1221
1222
 
1222
1223
  def normalize_results(
1223
1224
  self, results: RawSearchResult, **kwargs: Any
1224
- ) -> List[EOProduct]:
1225
+ ) -> list[EOProduct]:
1225
1226
  """Build :class:`~eodag.api.product._product.EOProduct` from provider result
1226
1227
 
1227
1228
  :param results: Raw provider result as single dict in list
@@ -1247,7 +1248,7 @@ class WekeoECMWFSearch(ECMWFSearch):
1247
1248
 
1248
1249
  return normalized
1249
1250
 
1250
- def do_search(self, *args: Any, **kwargs: Any) -> List[Dict[str, Any]]:
1251
+ def do_search(self, *args: Any, **kwargs: Any) -> list[dict[str, Any]]:
1251
1252
  """Should perform the actual search request.
1252
1253
 
1253
1254
  :param args: arguments to be used in the search
@@ -1258,7 +1259,7 @@ class WekeoECMWFSearch(ECMWFSearch):
1258
1259
 
1259
1260
  def build_query_string(
1260
1261
  self, product_type: str, **kwargs: Any
1261
- ) -> Tuple[Dict[str, Any], str]:
1262
+ ) -> tuple[dict[str, Any], str]:
1262
1263
  """Build The query string using the search parameters
1263
1264
 
1264
1265
  :param product_type: product type id
@@ -22,7 +22,7 @@ import logging
22
22
  import os
23
23
  import re
24
24
  from datetime import datetime
25
- from typing import TYPE_CHECKING, Any, Dict, List, Optional, Tuple, cast
25
+ from typing import TYPE_CHECKING, Any, Optional, cast
26
26
  from urllib.parse import urlsplit
27
27
 
28
28
  import boto3
@@ -69,8 +69,8 @@ def _get_date_from_yyyymmdd(date_str: str, item_key: str) -> Optional[datetime]:
69
69
 
70
70
 
71
71
  def _get_dates_from_dataset_data(
72
- dataset_item: Dict[str, Any]
73
- ) -> Optional[Dict[str, str]]:
72
+ dataset_item: dict[str, Any]
73
+ ) -> Optional[dict[str, str]]:
74
74
  dates = {}
75
75
  if "start_datetime" in dataset_item["properties"]:
76
76
  dates["start"] = dataset_item["properties"]["start_datetime"]
@@ -96,7 +96,7 @@ def _get_s3_client(endpoint_url: str) -> S3Client:
96
96
  )
97
97
 
98
98
 
99
- def _check_int_values_properties(properties: Dict[str, Any]):
99
+ def _check_int_values_properties(properties: dict[str, Any]):
100
100
  # remove int values with a bit length of more than 64 from the properties
101
101
  invalid = []
102
102
  for prop, prop_value in properties.items():
@@ -134,7 +134,7 @@ class CopMarineSearch(StaticStacSearch):
134
134
 
135
135
  def _get_product_type_info(
136
136
  self, product_type: str
137
- ) -> Tuple[Dict[str, Any], List[Dict[str, Any]]]:
137
+ ) -> tuple[dict[str, Any], list[dict[str, Any]]]:
138
138
  """Fetch product type and associated datasets info"""
139
139
 
140
140
  fetch_url = cast(str, self.config.discover_product_types["fetch_url"]).format(
@@ -183,13 +183,23 @@ class CopMarineSearch(StaticStacSearch):
183
183
  product_id: str,
184
184
  s3_url: str,
185
185
  product_type: str,
186
- dataset_item: Dict[str, Any],
187
- collection_dict: Dict[str, Any],
186
+ dataset_item: dict[str, Any],
187
+ collection_dict: dict[str, Any],
188
188
  ):
189
+ # try to find date(s) in product id
190
+ item_dates = re.findall(r"(\d{4})(0[1-9]|1[0-2])([0-3]\d)", product_id)
191
+ if not item_dates:
192
+ item_dates = re.findall(r"_(\d{4})(0[1-9]|1[0-2])", product_id)
193
+ use_dataset_dates = not bool(item_dates)
189
194
  for obj in collection_objects["Contents"]:
190
195
  if product_id in obj["Key"]:
191
196
  return self._create_product(
192
- product_type, obj["Key"], s3_url, dataset_item, collection_dict
197
+ product_type,
198
+ obj["Key"],
199
+ s3_url,
200
+ dataset_item,
201
+ collection_dict,
202
+ use_dataset_dates,
193
203
  )
194
204
  return None
195
205
 
@@ -198,8 +208,8 @@ class CopMarineSearch(StaticStacSearch):
198
208
  product_type: str,
199
209
  item_key: str,
200
210
  s3_url: str,
201
- dataset_item: Dict[str, Any],
202
- collection_dict: Dict[str, Any],
211
+ dataset_item: dict[str, Any],
212
+ collection_dict: dict[str, Any],
203
213
  use_dataset_dates: bool = False,
204
214
  ) -> Optional[EOProduct]:
205
215
 
@@ -278,7 +288,7 @@ class CopMarineSearch(StaticStacSearch):
278
288
  self,
279
289
  prep: PreparedSearch = PreparedSearch(),
280
290
  **kwargs: Any,
281
- ) -> Tuple[List[EOProduct], Optional[int]]:
291
+ ) -> tuple[list[EOProduct], Optional[int]]:
282
292
  """
283
293
  Implementation of search for the Copernicus Marine provider
284
294
  :param prep: object containing search parameterds
@@ -298,7 +308,7 @@ class CopMarineSearch(StaticStacSearch):
298
308
  "parameter product type is required for search with cop_marine provider"
299
309
  )
300
310
  collection_dict, datasets_items_list = self._get_product_type_info(product_type)
301
- products: List[EOProduct] = []
311
+ products: list[EOProduct] = []
302
312
  start_index = items_per_page * (page - 1) + 1
303
313
  num_total = 0
304
314
  for i, dataset_item in enumerate(datasets_items_list):
@@ -17,26 +17,16 @@
17
17
  # limitations under the License.
18
18
  import logging
19
19
  from types import MethodType
20
- from typing import Any, List
20
+ from typing import Any
21
21
 
22
- import boto3
23
- import botocore
24
22
  from botocore.exceptions import BotoCoreError
25
23
 
26
- from eodag.api.product import AssetsDict, EOProduct # type: ignore
24
+ from eodag.api.product import EOProduct # type: ignore
27
25
  from eodag.api.search_result import RawSearchResult
28
- from eodag.config import PluginConfig
29
- from eodag.plugins.authentication.aws_auth import AwsAuth
30
26
  from eodag.plugins.search.qssearch import ODataV4Search
31
- from eodag.utils import guess_file_type
32
- from eodag.utils.exceptions import (
33
- AuthenticationError,
34
- MisconfiguredError,
35
- NotAvailableError,
36
- RequestError,
37
- )
27
+ from eodag.utils.exceptions import RequestError
28
+ from eodag.utils.s3 import update_assets_from_s3
38
29
 
39
- DATA_EXTENSIONS = ["jp2", "tiff", "nc", "grib"]
40
30
  logger = logging.getLogger("eodag.search.creodiass3")
41
31
 
42
32
 
@@ -54,73 +44,13 @@ def patched_register_downloader(self, downloader, authenticator):
54
44
  self.register_downloader_only(downloader, authenticator)
55
45
  # and also update assets
56
46
  try:
57
- _update_assets(self, downloader.config, authenticator)
47
+ update_assets_from_s3(
48
+ self, authenticator, getattr(downloader.config, "s3_endpoint", None)
49
+ )
58
50
  except BotoCoreError as e:
59
51
  raise RequestError.from_error(e, "could not update assets") from e
60
52
 
61
53
 
62
- def _update_assets(product: EOProduct, config: PluginConfig, auth: AwsAuth):
63
- product.assets = AssetsDict(product)
64
- prefix = (
65
- product.properties.get("productIdentifier", None).replace("/eodata/", "") + "/"
66
- )
67
- if prefix:
68
- try:
69
- auth_dict = auth.authenticate()
70
- required_creds = ["aws_access_key_id", "aws_secret_access_key"]
71
- if not all(x in auth_dict for x in required_creds):
72
- raise MisconfiguredError(
73
- f"Incomplete credentials for {product.provider}, missing "
74
- f"{[x for x in required_creds if x not in auth_dict]}"
75
- )
76
- if not getattr(auth, "s3_client", None):
77
- auth.s3_client = boto3.client(
78
- "s3",
79
- endpoint_url=config.s3_endpoint,
80
- aws_access_key_id=auth_dict["aws_access_key_id"],
81
- aws_secret_access_key=auth_dict["aws_secret_access_key"],
82
- )
83
- logger.debug("Listing assets in %s", prefix)
84
- product.assets = AssetsDict(product)
85
- s3_res = auth.s3_client.list_objects(
86
- Bucket=config.s3_bucket, Prefix=prefix, MaxKeys=300
87
- )
88
- # check if product path has assets or is already a file
89
- if "Contents" in s3_res:
90
- for asset in s3_res["Contents"]:
91
- asset_basename = (
92
- asset["Key"].split("/")[-1]
93
- if "/" in asset["Key"]
94
- else asset["Key"]
95
- )
96
-
97
- if len(asset_basename) > 0 and asset_basename not in product.assets:
98
- role = (
99
- "data"
100
- if asset_basename.split(".")[-1] in DATA_EXTENSIONS
101
- else "metadata"
102
- )
103
-
104
- product.assets[asset_basename] = {
105
- "title": asset_basename,
106
- "roles": [role],
107
- "href": f"s3://{config.s3_bucket}/{asset['Key']}",
108
- }
109
- if mime_type := guess_file_type(asset["Key"]):
110
- product.assets[asset_basename]["type"] = mime_type
111
- # update driver
112
- product.driver = product.get_driver()
113
-
114
- except botocore.exceptions.ClientError as e:
115
- if str(auth.config.auth_error_code) in str(e):
116
- raise AuthenticationError(
117
- f"Authentication failed on {config.base_uri} s3"
118
- ) from e
119
- raise NotAvailableError(
120
- f"assets for product {prefix} could not be found"
121
- ) from e
122
-
123
-
124
54
  class CreodiasS3Search(ODataV4Search):
125
55
  """
126
56
  ``CreodiasS3Search`` is an extension of :class:`~eodag.plugins.search.qssearch.ODataV4Search`,
@@ -139,7 +69,7 @@ class CreodiasS3Search(ODataV4Search):
139
69
 
140
70
  def normalize_results(
141
71
  self, results: RawSearchResult, **kwargs: Any
142
- ) -> List[EOProduct]:
72
+ ) -> list[EOProduct]:
143
73
  """Build EOProducts from provider results"""
144
74
 
145
75
  products = super(CreodiasS3Search, self).normalize_results(results, **kwargs)
@@ -19,7 +19,7 @@ from __future__ import annotations
19
19
 
20
20
  import logging
21
21
  import re
22
- from typing import TYPE_CHECKING, Any, Dict, List, Optional, Tuple, Union
22
+ from typing import TYPE_CHECKING, Any, Optional, Union
23
23
 
24
24
  import pyproj
25
25
  from owslib.csw import CatalogueServiceWeb
@@ -60,13 +60,13 @@ class CSWSearch(Search):
60
60
  * :attr:`~eodag.config.PluginConfig.api_endpoint` (``str``) (**mandatory**): The endpoint of the
61
61
  provider's search interface
62
62
  * :attr:`~eodag.config.PluginConfig.version` (``str``): OGC Catalogue Service version; default: ``2.0.2``
63
- * :attr:`~eodag.config.PluginConfig.search_definition` (``Dict[str, Any]``) (**mandatory**):
63
+ * :attr:`~eodag.config.PluginConfig.search_definition` (``dict[str, Any]``) (**mandatory**):
64
64
 
65
- * **product_type_tags** (``List[Dict[str, Any]``): dict of product type tags
65
+ * **product_type_tags** (``list[dict[str, Any]``): dict of product type tags
66
66
  * **resource_location_filter** (``str``): regex string
67
- * **date_tags** (``Dict[str, Any]``): tags for start and end
67
+ * **date_tags** (``dict[str, Any]``): tags for start and end
68
68
 
69
- * :attr:`~eodag.config.PluginConfig.metadata_mapping` (``Dict[str, Any]``): The search plugins of this kind can
69
+ * :attr:`~eodag.config.PluginConfig.metadata_mapping` (``dict[str, Any]``): The search plugins of this kind can
70
70
  detect when a metadata mapping is "query-able", and get the semantics of how to format the query string
71
71
  parameter that enables to make a query on the corresponding metadata. To make a metadata query-able,
72
72
  just configure it in the metadata mapping to be a list of 2 items, the first one being the
@@ -107,7 +107,7 @@ class CSWSearch(Search):
107
107
  self,
108
108
  prep: PreparedSearch = PreparedSearch(),
109
109
  **kwargs: Any,
110
- ) -> Tuple[List[EOProduct], Optional[int]]:
110
+ ) -> tuple[list[EOProduct], Optional[int]]:
111
111
  """Perform a search on a OGC/CSW-like interface"""
112
112
  product_type = kwargs.get("productType")
113
113
  if product_type is None:
@@ -117,7 +117,7 @@ class CSWSearch(Search):
117
117
  self.__init_catalog(**getattr(auth.config, "credentials", {}))
118
118
  else:
119
119
  self.__init_catalog()
120
- results: List[EOProduct] = []
120
+ results: list[EOProduct] = []
121
121
  if self.catalog:
122
122
  provider_product_type = self.config.products[product_type]["productType"]
123
123
  for product_type_def in self.config.search_definition["product_type_tags"]:
@@ -229,12 +229,12 @@ class CSWSearch(Search):
229
229
 
230
230
  def __convert_query_params(
231
231
  self,
232
- product_type_def: Dict[str, Any],
232
+ product_type_def: dict[str, Any],
233
233
  product_type: str,
234
- params: Dict[str, Any],
235
- ) -> Union[List[OgcExpression], List[List[OgcExpression]]]:
234
+ params: dict[str, Any],
235
+ ) -> Union[list[OgcExpression], list[list[OgcExpression]]]:
236
236
  """Translates eodag search to CSW constraints using owslib constraint classes"""
237
- constraints: List[OgcExpression] = []
237
+ constraints: list[OgcExpression] = []
238
238
  # How the match should be performed (fuzzy, prefix, postfix or exact).
239
239
  # defaults to fuzzy
240
240
  pt_tag, matching = (