eodag 3.0.1__py3-none-any.whl → 3.1.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (87) hide show
  1. eodag/api/core.py +174 -138
  2. eodag/api/product/_assets.py +44 -15
  3. eodag/api/product/_product.py +58 -47
  4. eodag/api/product/drivers/__init__.py +81 -4
  5. eodag/api/product/drivers/base.py +65 -4
  6. eodag/api/product/drivers/generic.py +65 -0
  7. eodag/api/product/drivers/sentinel1.py +97 -0
  8. eodag/api/product/drivers/sentinel2.py +95 -0
  9. eodag/api/product/metadata_mapping.py +117 -90
  10. eodag/api/search_result.py +13 -23
  11. eodag/cli.py +26 -5
  12. eodag/config.py +86 -92
  13. eodag/plugins/apis/base.py +1 -1
  14. eodag/plugins/apis/ecmwf.py +42 -22
  15. eodag/plugins/apis/usgs.py +17 -16
  16. eodag/plugins/authentication/aws_auth.py +16 -13
  17. eodag/plugins/authentication/base.py +5 -3
  18. eodag/plugins/authentication/header.py +3 -3
  19. eodag/plugins/authentication/keycloak.py +4 -4
  20. eodag/plugins/authentication/oauth.py +7 -3
  21. eodag/plugins/authentication/openid_connect.py +22 -16
  22. eodag/plugins/authentication/sas_auth.py +4 -4
  23. eodag/plugins/authentication/token.py +41 -10
  24. eodag/plugins/authentication/token_exchange.py +1 -1
  25. eodag/plugins/base.py +4 -4
  26. eodag/plugins/crunch/base.py +4 -4
  27. eodag/plugins/crunch/filter_date.py +4 -4
  28. eodag/plugins/crunch/filter_latest_intersect.py +6 -6
  29. eodag/plugins/crunch/filter_latest_tpl_name.py +7 -7
  30. eodag/plugins/crunch/filter_overlap.py +4 -4
  31. eodag/plugins/crunch/filter_property.py +6 -7
  32. eodag/plugins/download/aws.py +146 -87
  33. eodag/plugins/download/base.py +38 -56
  34. eodag/plugins/download/creodias_s3.py +29 -0
  35. eodag/plugins/download/http.py +173 -183
  36. eodag/plugins/download/s3rest.py +10 -11
  37. eodag/plugins/manager.py +10 -20
  38. eodag/plugins/search/__init__.py +6 -5
  39. eodag/plugins/search/base.py +90 -46
  40. eodag/plugins/search/build_search_result.py +1048 -361
  41. eodag/plugins/search/cop_marine.py +22 -12
  42. eodag/plugins/search/creodias_s3.py +9 -73
  43. eodag/plugins/search/csw.py +11 -11
  44. eodag/plugins/search/data_request_search.py +19 -18
  45. eodag/plugins/search/qssearch.py +99 -258
  46. eodag/plugins/search/stac_list_assets.py +85 -0
  47. eodag/plugins/search/static_stac_search.py +4 -4
  48. eodag/resources/ext_product_types.json +1 -1
  49. eodag/resources/product_types.yml +1134 -325
  50. eodag/resources/providers.yml +906 -2006
  51. eodag/resources/stac_api.yml +2 -2
  52. eodag/resources/user_conf_template.yml +10 -9
  53. eodag/rest/cache.py +2 -2
  54. eodag/rest/config.py +3 -3
  55. eodag/rest/core.py +112 -82
  56. eodag/rest/errors.py +5 -5
  57. eodag/rest/server.py +33 -14
  58. eodag/rest/stac.py +41 -38
  59. eodag/rest/types/collections_search.py +3 -3
  60. eodag/rest/types/eodag_search.py +29 -23
  61. eodag/rest/types/queryables.py +42 -31
  62. eodag/rest/types/stac_search.py +15 -25
  63. eodag/rest/utils/__init__.py +14 -21
  64. eodag/rest/utils/cql_evaluate.py +6 -6
  65. eodag/rest/utils/rfc3339.py +2 -2
  66. eodag/types/__init__.py +141 -32
  67. eodag/types/bbox.py +2 -2
  68. eodag/types/download_args.py +3 -3
  69. eodag/types/queryables.py +183 -72
  70. eodag/types/search_args.py +4 -4
  71. eodag/types/whoosh.py +127 -3
  72. eodag/utils/__init__.py +153 -51
  73. eodag/utils/exceptions.py +28 -21
  74. eodag/utils/import_system.py +2 -2
  75. eodag/utils/repr.py +65 -6
  76. eodag/utils/requests.py +13 -13
  77. eodag/utils/rest.py +2 -2
  78. eodag/utils/s3.py +231 -0
  79. eodag/utils/stac_reader.py +10 -10
  80. {eodag-3.0.1.dist-info → eodag-3.1.0.dist-info}/METADATA +77 -76
  81. eodag-3.1.0.dist-info/RECORD +113 -0
  82. {eodag-3.0.1.dist-info → eodag-3.1.0.dist-info}/WHEEL +1 -1
  83. {eodag-3.0.1.dist-info → eodag-3.1.0.dist-info}/entry_points.txt +4 -2
  84. eodag/utils/constraints.py +0 -244
  85. eodag-3.0.1.dist-info/RECORD +0 -109
  86. {eodag-3.0.1.dist-info → eodag-3.1.0.dist-info}/LICENSE +0 -0
  87. {eodag-3.0.1.dist-info → eodag-3.1.0.dist-info}/top_level.txt +0 -0
eodag/api/core.py CHANGED
@@ -23,28 +23,16 @@ import os
23
23
  import re
24
24
  import shutil
25
25
  import tempfile
26
+ from importlib.metadata import version
27
+ from importlib.resources import files as res_files
26
28
  from operator import itemgetter
27
- from typing import (
28
- TYPE_CHECKING,
29
- Annotated,
30
- Any,
31
- Dict,
32
- Iterator,
33
- List,
34
- Optional,
35
- Set,
36
- Tuple,
37
- Union,
38
- )
29
+ from typing import TYPE_CHECKING, Any, Iterator, Optional, Union
39
30
 
40
31
  import geojson
41
- import pkg_resources
42
32
  import yaml.parser
43
- from pkg_resources import resource_filename
44
- from pydantic.fields import FieldInfo
45
33
  from whoosh import analysis, fields
46
34
  from whoosh.fields import Schema
47
- from whoosh.index import create_in, exists_in, open_dir
35
+ from whoosh.index import exists_in, open_dir
48
36
  from whoosh.qparser import QueryParser
49
37
 
50
38
  from eodag.api.product.metadata_mapping import (
@@ -69,11 +57,11 @@ from eodag.config import (
69
57
  )
70
58
  from eodag.plugins.manager import PluginManager
71
59
  from eodag.plugins.search import PreparedSearch
72
- from eodag.plugins.search.build_search_result import BuildPostSearchResult
60
+ from eodag.plugins.search.build_search_result import MeteoblueSearch
73
61
  from eodag.plugins.search.qssearch import PostJsonSearch
74
62
  from eodag.types import model_fields_to_annotated
75
- from eodag.types.queryables import CommonQueryables
76
- from eodag.types.whoosh import EODAGQueryParser
63
+ from eodag.types.queryables import CommonQueryables, QueryablesDict
64
+ from eodag.types.whoosh import EODAGQueryParser, create_in
77
65
  from eodag.utils import (
78
66
  DEFAULT_DOWNLOAD_TIMEOUT,
79
67
  DEFAULT_DOWNLOAD_WAIT,
@@ -84,7 +72,6 @@ from eodag.utils import (
84
72
  HTTP_REQ_TIMEOUT,
85
73
  MockResponse,
86
74
  _deprecated,
87
- copy_deepcopy,
88
75
  get_geometry_from_various,
89
76
  makedirs,
90
77
  obj_md5sum,
@@ -93,6 +80,7 @@ from eodag.utils import (
93
80
  uri_to_path,
94
81
  )
95
82
  from eodag.utils.exceptions import (
83
+ AuthenticationError,
96
84
  EodagError,
97
85
  NoMatchingProductType,
98
86
  PluginImplementationError,
@@ -131,8 +119,8 @@ class EODataAccessGateway:
131
119
  user_conf_file_path: Optional[str] = None,
132
120
  locations_conf_path: Optional[str] = None,
133
121
  ) -> None:
134
- product_types_config_path = resource_filename(
135
- "eodag", os.path.join("resources/", "product_types.yml")
122
+ product_types_config_path = os.getenv("EODAG_PRODUCT_TYPES_CFG_FILE") or str(
123
+ res_files("eodag") / "resources" / "product_types.yml"
136
124
  )
137
125
  self.product_types_config = SimpleYamlProxyConfig(product_types_config_path)
138
126
  self.product_types_config_md5 = obj_md5sum(self.product_types_config.source)
@@ -173,8 +161,8 @@ class EODataAccessGateway:
173
161
  user_conf_file_path = standard_configuration_path
174
162
  if not os.path.isfile(standard_configuration_path):
175
163
  shutil.copy(
176
- resource_filename(
177
- "eodag", os.path.join("resources", "user_conf_template.yml")
164
+ str(
165
+ res_files("eodag") / "resources" / "user_conf_template.yml"
178
166
  ),
179
167
  standard_configuration_path,
180
168
  )
@@ -197,7 +185,7 @@ class EODataAccessGateway:
197
185
  self._plugins_manager.rebuild(self.providers_config)
198
186
 
199
187
  # store pruned providers configs
200
- self._pruned_providers_config: Dict[str, Any] = {}
188
+ self._pruned_providers_config: dict[str, Any] = {}
201
189
  # filter out providers needing auth that have no credentials set
202
190
  self._prune_providers_list()
203
191
 
@@ -215,13 +203,13 @@ class EODataAccessGateway:
215
203
  locations_conf_path = os.path.join(self.conf_dir, "locations.yml")
216
204
  if not os.path.isfile(locations_conf_path):
217
205
  # copy locations conf file and replace path example
218
- locations_conf_template = resource_filename(
219
- "eodag",
220
- os.path.join("resources", "locations_conf_template.yml"),
206
+ locations_conf_template = str(
207
+ res_files("eodag") / "resources" / "locations_conf_template.yml"
221
208
  )
222
- with open(locations_conf_template) as infile, open(
223
- locations_conf_path, "w"
224
- ) as outfile:
209
+ with (
210
+ open(locations_conf_template) as infile,
211
+ open(locations_conf_path, "w") as outfile,
212
+ ):
225
213
  # The template contains paths in the form of:
226
214
  # /path/to/locations/file.shp
227
215
  path_template = "/path/to/locations/"
@@ -233,14 +221,14 @@ class EODataAccessGateway:
233
221
  outfile.write(line)
234
222
  # copy sample shapefile dir
235
223
  shutil.copytree(
236
- resource_filename("eodag", os.path.join("resources", "shp")),
224
+ str(res_files("eodag") / "resources" / "shp"),
237
225
  os.path.join(self.conf_dir, "shp"),
238
226
  )
239
227
  self.set_locations_conf(locations_conf_path)
240
228
 
241
229
  def get_version(self) -> str:
242
230
  """Get eodag package version"""
243
- return pkg_resources.get_distribution("eodag").version
231
+ return version("eodag")
244
232
 
245
233
  def build_index(self) -> None:
246
234
  """Build a `Whoosh <https://whoosh.readthedocs.io/en/latest/index.html>`_
@@ -317,13 +305,18 @@ class EODataAccessGateway:
317
305
  product_type, **{"md5": self.product_types_config_md5}
318
306
  )
319
307
  # add to index
320
- ix_writer.add_document(
321
- **{
322
- k: v
323
- for k, v in versioned_product_type.items()
324
- if k in product_types_schema.names()
325
- }
326
- )
308
+ try:
309
+ ix_writer.add_document(
310
+ **{
311
+ k: v
312
+ for k, v in versioned_product_type.items()
313
+ if k in product_types_schema.names()
314
+ }
315
+ )
316
+ except TypeError as e:
317
+ logger.error(
318
+ f"Cannot write product type {product_type['ID']} into index. e={e} product_type={product_type}"
319
+ )
327
320
  ix_writer.commit()
328
321
 
329
322
  def set_preferred_provider(self, provider: str) -> None:
@@ -341,7 +334,7 @@ class EODataAccessGateway:
341
334
  new_priority = max_priority + 1
342
335
  self._plugins_manager.set_priority(provider, new_priority)
343
336
 
344
- def get_preferred_provider(self) -> Tuple[str, int]:
337
+ def get_preferred_provider(self) -> tuple[str, int]:
345
338
  """Get the provider currently set as the preferred one for searching
346
339
  products, along with its priority.
347
340
 
@@ -357,7 +350,7 @@ class EODataAccessGateway:
357
350
  def update_providers_config(
358
351
  self,
359
352
  yaml_conf: Optional[str] = None,
360
- dict_conf: Optional[Dict[str, Any]] = None,
353
+ dict_conf: Optional[dict[str, Any]] = None,
361
354
  ) -> None:
362
355
  """Update providers configuration with given input.
363
356
  Can be used to add a provider to existing configuration or update
@@ -403,12 +396,12 @@ class EODataAccessGateway:
403
396
  name: str,
404
397
  url: Optional[str] = None,
405
398
  priority: Optional[int] = None,
406
- search: Dict[str, Any] = {"type": "StacSearch"},
407
- products: Dict[str, Any] = {
399
+ search: dict[str, Any] = {"type": "StacSearch"},
400
+ products: dict[str, Any] = {
408
401
  GENERIC_PRODUCT_TYPE: {"productType": "{productType}"}
409
402
  },
410
- download: Dict[str, Any] = {"type": "HTTPDownload", "auth_error_code": 401},
411
- **kwargs: Dict[str, Any],
403
+ download: dict[str, Any] = {"type": "HTTPDownload", "auth_error_code": 401},
404
+ **kwargs: dict[str, Any],
412
405
  ):
413
406
  """Adds a new provider.
414
407
 
@@ -427,7 +420,7 @@ class EODataAccessGateway:
427
420
  :param download: Download :class:`~eodag.config.PluginConfig` mapping
428
421
  :param kwargs: Additional :class:`~eodag.config.ProviderConfig` mapping
429
422
  """
430
- conf_dict: Dict[str, Any] = {
423
+ conf_dict: dict[str, Any] = {
431
424
  name: {
432
425
  "url": url,
433
426
  "search": {"type": "StacSearch", **search},
@@ -571,7 +564,7 @@ class EODataAccessGateway:
571
564
  main_locations_config = locations_config[main_key]
572
565
 
573
566
  logger.info("Locations configuration loaded from %s" % locations_conf_path)
574
- self.locations_config: List[Dict[str, Any]] = main_locations_config
567
+ self.locations_config: list[dict[str, Any]] = main_locations_config
575
568
  else:
576
569
  logger.info(
577
570
  "Could not load locations configuration from %s" % locations_conf_path
@@ -580,7 +573,7 @@ class EODataAccessGateway:
580
573
 
581
574
  def list_product_types(
582
575
  self, provider: Optional[str] = None, fetch_providers: bool = True
583
- ) -> List[Dict[str, Any]]:
576
+ ) -> list[dict[str, Any]]:
584
577
  """Lists supported product types.
585
578
 
586
579
  :param provider: (optional) The name of a provider that must support the product
@@ -594,7 +587,7 @@ class EODataAccessGateway:
594
587
  # First, update product types list if possible
595
588
  self.fetch_product_types_list(provider=provider)
596
589
 
597
- product_types: List[Dict[str, Any]] = []
590
+ product_types: list[dict[str, Any]] = []
598
591
 
599
592
  providers_configs = (
600
593
  list(self.providers_config.values())
@@ -650,7 +643,7 @@ class EODataAccessGateway:
650
643
  providers_to_fetch = [provider]
651
644
 
652
645
  # providers discovery confs that are fetchable
653
- providers_discovery_configs_fetchable: Dict[str, Any] = {}
646
+ providers_discovery_configs_fetchable: dict[str, Any] = {}
654
647
  # check if any provider has not already been fetched for product types
655
648
  already_fetched = True
656
649
  for provider_to_fetch in providers_to_fetch:
@@ -773,7 +766,7 @@ class EODataAccessGateway:
773
766
 
774
767
  def discover_product_types(
775
768
  self, provider: Optional[str] = None
776
- ) -> Optional[Dict[str, Any]]:
769
+ ) -> Optional[dict[str, Any]]:
777
770
  """Fetch providers for product types
778
771
 
779
772
  :param provider: The name of a provider or provider-group to fetch. Defaults to
@@ -793,7 +786,7 @@ class EODataAccessGateway:
793
786
  raise UnsupportedProvider(
794
787
  f"The requested provider is not (yet) supported: {provider}"
795
788
  )
796
- ext_product_types_conf: Dict[str, Any] = {}
789
+ ext_product_types_conf: dict[str, Any] = {}
797
790
  providers_to_fetch = [
798
791
  p
799
792
  for p in (
@@ -806,7 +799,7 @@ class EODataAccessGateway:
806
799
  else self.available_providers()
807
800
  )
808
801
  ]
809
- kwargs: Dict[str, Any] = {}
802
+ kwargs: dict[str, Any] = {}
810
803
  for provider in providers_to_fetch:
811
804
  if hasattr(self.providers_config[provider], "search"):
812
805
  search_plugin_config = self.providers_config[provider].search
@@ -847,7 +840,7 @@ class EODataAccessGateway:
847
840
  return sort_dict(ext_product_types_conf)
848
841
 
849
842
  def update_product_types_list(
850
- self, ext_product_types_conf: Dict[str, Optional[Dict[str, Dict[str, Any]]]]
843
+ self, ext_product_types_conf: dict[str, Optional[dict[str, dict[str, Any]]]]
851
844
  ) -> None:
852
845
  """Update eodag product types list
853
846
 
@@ -875,7 +868,7 @@ class EODataAccessGateway:
875
868
  provider,
876
869
  )
877
870
  continue
878
- new_product_types: List[str] = []
871
+ new_product_types: list[str] = []
879
872
  for (
880
873
  new_product_type,
881
874
  new_product_type_conf,
@@ -938,7 +931,7 @@ class EODataAccessGateway:
938
931
 
939
932
  def available_providers(
940
933
  self, product_type: Optional[str] = None, by_group: bool = False
941
- ) -> List[str]:
934
+ ) -> list[str]:
942
935
  """Gives the sorted list of the available providers or groups
943
936
 
944
937
  The providers or groups are sorted first by their priority level in descending order,
@@ -965,7 +958,7 @@ class EODataAccessGateway:
965
958
 
966
959
  # If by_group is True, keep only the highest priority for each group
967
960
  if by_group:
968
- group_priority: Dict[str, int] = {}
961
+ group_priority: dict[str, int] = {}
969
962
  for name, priority in providers:
970
963
  if name not in group_priority or priority > group_priority[name]:
971
964
  group_priority[name] = priority
@@ -1032,7 +1025,7 @@ class EODataAccessGateway:
1032
1025
  missionStartDate: Optional[str] = None,
1033
1026
  missionEndDate: Optional[str] = None,
1034
1027
  **kwargs: Any,
1035
- ) -> List[str]:
1028
+ ) -> list[str]:
1036
1029
  """
1037
1030
  Find EODAG product type IDs that best match a set of search parameters.
1038
1031
 
@@ -1090,7 +1083,7 @@ class EODataAccessGateway:
1090
1083
  query = p.parse(text)
1091
1084
  results = searcher.search(query, limit=None)
1092
1085
 
1093
- guesses: List[Dict[str, str]] = [dict(r) for r in results or []]
1086
+ guesses: list[dict[str, str]] = [dict(r) for r in results or []]
1094
1087
 
1095
1088
  # datetime filtering
1096
1089
  if missionStartDate or missionEndDate:
@@ -1131,8 +1124,8 @@ class EODataAccessGateway:
1131
1124
  raise_errors: bool = False,
1132
1125
  start: Optional[str] = None,
1133
1126
  end: Optional[str] = None,
1134
- geom: Optional[Union[str, Dict[str, float], BaseGeometry]] = None,
1135
- locations: Optional[Dict[str, str]] = None,
1127
+ geom: Optional[Union[str, dict[str, float], BaseGeometry]] = None,
1128
+ locations: Optional[dict[str, str]] = None,
1136
1129
  provider: Optional[str] = None,
1137
1130
  count: bool = False,
1138
1131
  **kwargs: Any,
@@ -1211,7 +1204,7 @@ class EODataAccessGateway:
1211
1204
  items_per_page=items_per_page,
1212
1205
  )
1213
1206
 
1214
- errors: List[Tuple[str, Exception]] = []
1207
+ errors: list[tuple[str, Exception]] = []
1215
1208
  # Loop over available providers and return the first non-empty results
1216
1209
  for i, search_plugin in enumerate(search_plugins):
1217
1210
  search_plugin.clear()
@@ -1240,8 +1233,8 @@ class EODataAccessGateway:
1240
1233
  items_per_page: int = DEFAULT_ITEMS_PER_PAGE,
1241
1234
  start: Optional[str] = None,
1242
1235
  end: Optional[str] = None,
1243
- geom: Optional[Union[str, Dict[str, float], BaseGeometry]] = None,
1244
- locations: Optional[Dict[str, str]] = None,
1236
+ geom: Optional[Union[str, dict[str, float], BaseGeometry]] = None,
1237
+ locations: Optional[dict[str, str]] = None,
1245
1238
  **kwargs: Any,
1246
1239
  ) -> Iterator[SearchResult]:
1247
1240
  """Iterate over the pages of a products search.
@@ -1417,8 +1410,8 @@ class EODataAccessGateway:
1417
1410
  items_per_page: Optional[int] = None,
1418
1411
  start: Optional[str] = None,
1419
1412
  end: Optional[str] = None,
1420
- geom: Optional[Union[str, Dict[str, float], BaseGeometry]] = None,
1421
- locations: Optional[Dict[str, str]] = None,
1413
+ geom: Optional[Union[str, dict[str, float], BaseGeometry]] = None,
1414
+ locations: Optional[dict[str, str]] = None,
1422
1415
  **kwargs: Any,
1423
1416
  ) -> SearchResult:
1424
1417
  """Search and return all the products matching the search criteria.
@@ -1639,7 +1632,7 @@ class EODataAccessGateway:
1639
1632
  if not getattr(plugin.config, "discover_product_types", {}).get("fetch_url"):
1640
1633
  return None
1641
1634
 
1642
- kwargs: Dict[str, Any] = {"productType": product_type}
1635
+ kwargs: dict[str, Any] = {"productType": product_type}
1643
1636
 
1644
1637
  # append auth if needed
1645
1638
  if getattr(plugin.config, "need_auth", False):
@@ -1657,11 +1650,11 @@ class EODataAccessGateway:
1657
1650
  self,
1658
1651
  start: Optional[str] = None,
1659
1652
  end: Optional[str] = None,
1660
- geom: Optional[Union[str, Dict[str, float], BaseGeometry]] = None,
1661
- locations: Optional[Dict[str, str]] = None,
1653
+ geom: Optional[Union[str, dict[str, float], BaseGeometry]] = None,
1654
+ locations: Optional[dict[str, str]] = None,
1662
1655
  provider: Optional[str] = None,
1663
1656
  **kwargs: Any,
1664
- ) -> Tuple[List[Union[Search, Api]], Dict[str, Any]]:
1657
+ ) -> tuple[list[Union[Search, Api]], dict[str, Any]]:
1665
1658
  """Internal method to prepare the search kwargs and get the search plugins.
1666
1659
 
1667
1660
  Product query:
@@ -1769,16 +1762,16 @@ class EODataAccessGateway:
1769
1762
 
1770
1763
  preferred_provider = self.get_preferred_provider()[0]
1771
1764
 
1772
- search_plugins: List[Union[Search, Api]] = []
1765
+ search_plugins: list[Union[Search, Api]] = []
1773
1766
  for plugin in self._plugins_manager.get_search_plugins(
1774
1767
  product_type=product_type, provider=provider
1775
1768
  ):
1776
- # exclude BuildPostSearchResult plugins from search fallback for unknow product_type
1769
+ # exclude MeteoblueSearch plugins from search fallback for unknown product_type
1777
1770
  if (
1778
1771
  provider != plugin.provider
1779
1772
  and preferred_provider != plugin.provider
1780
1773
  and product_type not in self.product_types_config
1781
- and isinstance(plugin, BuildPostSearchResult)
1774
+ and isinstance(plugin, MeteoblueSearch)
1782
1775
  ):
1783
1776
  continue
1784
1777
  search_plugins.append(plugin)
@@ -1801,27 +1794,7 @@ class EODataAccessGateway:
1801
1794
  # Add product_types_config to plugin config. This dict contains product
1802
1795
  # type metadata that will also be stored in each product's properties.
1803
1796
  for search_plugin in search_plugins:
1804
- try:
1805
- search_plugin.config.product_type_config = dict(
1806
- [
1807
- p
1808
- for p in self.list_product_types(
1809
- search_plugin.provider, fetch_providers=False
1810
- )
1811
- if p["_id"] == product_type
1812
- ][0],
1813
- **{"productType": product_type},
1814
- )
1815
- # If the product isn't in the catalog, it's a generic product type.
1816
- except IndexError:
1817
- # Construct the GENERIC_PRODUCT_TYPE metadata
1818
- search_plugin.config.product_type_config = dict(
1819
- ID=GENERIC_PRODUCT_TYPE,
1820
- **self.product_types_config[GENERIC_PRODUCT_TYPE],
1821
- productType=product_type,
1822
- )
1823
- # Remove the ID since this is equal to productType.
1824
- search_plugin.config.product_type_config.pop("ID", None)
1797
+ self._attach_product_type_config(search_plugin, product_type)
1825
1798
 
1826
1799
  return search_plugins, kwargs
1827
1800
 
@@ -1859,10 +1832,10 @@ class EODataAccessGateway:
1859
1832
  max_items_per_page,
1860
1833
  )
1861
1834
 
1862
- results: List[EOProduct] = []
1835
+ results: list[EOProduct] = []
1863
1836
  total_results: Optional[int] = 0 if count else None
1864
1837
 
1865
- errors: List[Tuple[str, Exception]] = []
1838
+ errors: list[tuple[str, Exception]] = []
1866
1839
 
1867
1840
  try:
1868
1841
  prep = PreparedSearch(count=count)
@@ -2010,7 +1983,7 @@ class EODataAccessGateway:
2010
1983
  return results
2011
1984
 
2012
1985
  @staticmethod
2013
- def group_by_extent(searches: List[SearchResult]) -> List[SearchResult]:
1986
+ def group_by_extent(searches: list[SearchResult]) -> list[SearchResult]:
2014
1987
  """Combines multiple SearchResults and return a list of SearchResults grouped
2015
1988
  by extent (i.e. bounding box).
2016
1989
 
@@ -2019,7 +1992,7 @@ class EODataAccessGateway:
2019
1992
  """
2020
1993
  # Dict with extents as keys, each extent being defined by a str
2021
1994
  # "{minx}{miny}{maxx}{maxy}" (each float rounded to 2 dec).
2022
- products_grouped_by_extent: Dict[str, Any] = {}
1995
+ products_grouped_by_extent: dict[str, Any] = {}
2023
1996
 
2024
1997
  for search in searches:
2025
1998
  for product in search:
@@ -2038,10 +2011,10 @@ class EODataAccessGateway:
2038
2011
  search_result: SearchResult,
2039
2012
  downloaded_callback: Optional[DownloadedCallback] = None,
2040
2013
  progress_callback: Optional[ProgressCallback] = None,
2041
- wait: int = DEFAULT_DOWNLOAD_WAIT,
2042
- timeout: int = DEFAULT_DOWNLOAD_TIMEOUT,
2014
+ wait: float = DEFAULT_DOWNLOAD_WAIT,
2015
+ timeout: float = DEFAULT_DOWNLOAD_TIMEOUT,
2043
2016
  **kwargs: Unpack[DownloadConf],
2044
- ) -> List[str]:
2017
+ ) -> list[str]:
2045
2018
  """Download all products resulting from a search.
2046
2019
 
2047
2020
  :param search_result: A collection of EO products resulting from a search
@@ -2201,8 +2174,8 @@ class EODataAccessGateway:
2201
2174
  self,
2202
2175
  product: EOProduct,
2203
2176
  progress_callback: Optional[ProgressCallback] = None,
2204
- wait: int = DEFAULT_DOWNLOAD_WAIT,
2205
- timeout: int = DEFAULT_DOWNLOAD_TIMEOUT,
2177
+ wait: float = DEFAULT_DOWNLOAD_WAIT,
2178
+ timeout: float = DEFAULT_DOWNLOAD_TIMEOUT,
2206
2179
  **kwargs: Unpack[DownloadConf],
2207
2180
  ) -> str:
2208
2181
  """Download a single product.
@@ -2280,71 +2253,107 @@ class EODataAccessGateway:
2280
2253
  return self._plugins_manager.get_crunch_plugin(name, **plugin_conf)
2281
2254
 
2282
2255
  def list_queryables(
2283
- self, provider: Optional[str] = None, **kwargs: Any
2284
- ) -> Dict[str, Annotated[Any, FieldInfo]]:
2256
+ self,
2257
+ provider: Optional[str] = None,
2258
+ fetch_providers: bool = True,
2259
+ **kwargs: Any,
2260
+ ) -> QueryablesDict:
2285
2261
  """Fetch the queryable properties for a given product type and/or provider.
2286
2262
 
2287
2263
  :param provider: (optional) The provider.
2264
+ :param fetch_providers: If new product types should be fetched from the providers; default: True
2288
2265
  :param kwargs: additional filters for queryables (`productType` or other search
2289
2266
  arguments)
2290
2267
 
2291
2268
  :raises UnsupportedProductType: If the specified product type is not available for the
2292
2269
  provider.
2293
2270
 
2294
- :returns: A dict containing the EODAG queryable properties, associating
2295
- parameters to their annotated type
2271
+ :returns: A :class:`~eodag.api.product.queryables.QuerybalesDict` containing the EODAG queryable
2272
+ properties, associating parameters to their annotated type, and a additional_properties attribute
2296
2273
  """
2297
- available_product_types = [
2274
+ # only fetch providers if product type is not found
2275
+ available_product_types: list[str] = [
2298
2276
  pt["ID"]
2299
2277
  for pt in self.list_product_types(provider=provider, fetch_providers=False)
2300
2278
  ]
2301
- product_type = kwargs.get("productType")
2279
+ product_type: Optional[str] = kwargs.get("productType")
2280
+ pt_alias: Optional[str] = product_type
2302
2281
 
2303
2282
  if product_type:
2283
+ if product_type not in available_product_types:
2284
+ if fetch_providers:
2285
+ # fetch providers and try again
2286
+ available_product_types = [
2287
+ pt["ID"]
2288
+ for pt in self.list_product_types(
2289
+ provider=provider, fetch_providers=True
2290
+ )
2291
+ ]
2292
+ raise UnsupportedProductType(f"{product_type} is not available.")
2304
2293
  try:
2305
2294
  kwargs["productType"] = product_type = self.get_product_type_from_alias(
2306
2295
  product_type
2307
2296
  )
2308
2297
  except NoMatchingProductType as e:
2309
- raise UnsupportedProductType(f"{product_type} is not available") from e
2310
-
2311
- if product_type and product_type not in available_product_types:
2312
- self.fetch_product_types_list()
2298
+ raise UnsupportedProductType(f"{product_type} is not available.") from e
2313
2299
 
2314
2300
  if not provider and not product_type:
2315
- return model_fields_to_annotated(CommonQueryables.model_fields)
2301
+ return QueryablesDict(
2302
+ additional_properties=True,
2303
+ **model_fields_to_annotated(CommonQueryables.model_fields),
2304
+ )
2316
2305
 
2317
- providers_queryables: Dict[str, Dict[str, Annotated[Any, FieldInfo]]] = {}
2306
+ additional_properties = False
2307
+ additional_information = []
2308
+ queryable_properties: dict[str, Any] = {}
2318
2309
 
2319
2310
  for plugin in self._plugins_manager.get_search_plugins(product_type, provider):
2311
+ # attach product type config
2312
+ product_type_configs: dict[str, Any] = {}
2313
+ if product_type:
2314
+ self._attach_product_type_config(plugin, product_type)
2315
+ product_type_configs[product_type] = plugin.config.product_type_config
2316
+ else:
2317
+ for pt in available_product_types:
2318
+ self._attach_product_type_config(plugin, pt)
2319
+ product_type_configs[pt] = plugin.config.product_type_config
2320
+
2321
+ # authenticate if required
2320
2322
  if getattr(plugin.config, "need_auth", False) and (
2321
2323
  auth := self._plugins_manager.get_auth_plugin(plugin)
2322
2324
  ):
2323
- plugin.auth = auth.authenticate()
2324
- providers_queryables[plugin.provider] = plugin.list_queryables(
2325
- filters=kwargs, product_type=product_type
2326
- )
2327
-
2328
- queryable_keys: Set[str] = set.intersection( # type: ignore
2329
- *[set(q.keys()) for q in providers_queryables.values()]
2330
- )
2331
- queryables = {
2332
- k: v
2333
- for k, v in list(providers_queryables.values())[0].items()
2334
- if k in queryable_keys
2335
- }
2325
+ try:
2326
+ plugin.auth = auth.authenticate()
2327
+ except AuthenticationError:
2328
+ logger.debug(
2329
+ "queryables from provider %s could not be fetched due to an authentication error",
2330
+ plugin.provider,
2331
+ )
2336
2332
 
2337
- # always keep at least CommonQueryables
2338
- common_queryables = copy_deepcopy(CommonQueryables.model_fields)
2339
- for key, queryable in common_queryables.items():
2340
- if key in kwargs:
2341
- queryable.default = kwargs[key]
2333
+ plugin_queryables = plugin.list_queryables(
2334
+ kwargs,
2335
+ available_product_types,
2336
+ product_type_configs,
2337
+ product_type,
2338
+ pt_alias,
2339
+ )
2342
2340
 
2343
- queryables.update(model_fields_to_annotated(common_queryables))
2341
+ if plugin_queryables.additional_information:
2342
+ additional_information.append(
2343
+ f"{plugin.provider}: {plugin_queryables.additional_information}"
2344
+ )
2345
+ queryable_properties = {**plugin_queryables, **queryable_properties}
2346
+ additional_properties = (
2347
+ additional_properties or plugin_queryables.additional_properties
2348
+ )
2344
2349
 
2345
- return queryables
2350
+ return QueryablesDict(
2351
+ additional_properties=additional_properties,
2352
+ additional_information=" | ".join(additional_information),
2353
+ **queryable_properties,
2354
+ )
2346
2355
 
2347
- def available_sortables(self) -> Dict[str, Optional[ProviderSortables]]:
2356
+ def available_sortables(self) -> dict[str, Optional[ProviderSortables]]:
2348
2357
  """For each provider, gives its available sortable parameter(s) and its maximum
2349
2358
  number of them if it supports the sorting feature, otherwise gives None.
2350
2359
 
@@ -2352,7 +2361,7 @@ class EODataAccessGateway:
2352
2361
  its (their) maximum number as value(s).
2353
2362
  :raises: :class:`~eodag.utils.exceptions.UnsupportedProvider`
2354
2363
  """
2355
- sortables: Dict[str, Optional[ProviderSortables]] = {}
2364
+ sortables: dict[str, Optional[ProviderSortables]] = {}
2356
2365
  provider_search_plugins = self._plugins_manager.get_search_plugins()
2357
2366
  for provider_search_plugin in provider_search_plugins:
2358
2367
  provider = provider_search_plugin.provider
@@ -2375,3 +2384,30 @@ class EODataAccessGateway:
2375
2384
  ],
2376
2385
  }
2377
2386
  return sortables
2387
+
2388
+ def _attach_product_type_config(self, plugin: Search, product_type: str) -> None:
2389
+ """
2390
+ Attach product_types_config to plugin config. This dict contains product
2391
+ type metadata that will also be stored in each product's properties.
2392
+ """
2393
+ try:
2394
+ plugin.config.product_type_config = dict(
2395
+ [
2396
+ p
2397
+ for p in self.list_product_types(
2398
+ plugin.provider, fetch_providers=False
2399
+ )
2400
+ if p["_id"] == product_type
2401
+ ][0],
2402
+ **{"productType": product_type},
2403
+ )
2404
+ # If the product isn't in the catalog, it's a generic product type.
2405
+ except IndexError:
2406
+ # Construct the GENERIC_PRODUCT_TYPE metadata
2407
+ plugin.config.product_type_config = dict(
2408
+ ID=GENERIC_PRODUCT_TYPE,
2409
+ **self.product_types_config[GENERIC_PRODUCT_TYPE],
2410
+ productType=product_type,
2411
+ )
2412
+ # Remove the ID since this is equal to productType.
2413
+ plugin.config.product_type_config.pop("ID", None)