eodag 3.0.1__py3-none-any.whl → 3.1.0b2__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (87) hide show
  1. eodag/api/core.py +164 -127
  2. eodag/api/product/_assets.py +11 -11
  3. eodag/api/product/_product.py +45 -30
  4. eodag/api/product/drivers/__init__.py +81 -4
  5. eodag/api/product/drivers/base.py +65 -4
  6. eodag/api/product/drivers/generic.py +65 -0
  7. eodag/api/product/drivers/sentinel1.py +97 -0
  8. eodag/api/product/drivers/sentinel2.py +95 -0
  9. eodag/api/product/metadata_mapping.py +101 -85
  10. eodag/api/search_result.py +13 -23
  11. eodag/cli.py +26 -5
  12. eodag/config.py +78 -81
  13. eodag/plugins/apis/base.py +1 -1
  14. eodag/plugins/apis/ecmwf.py +46 -22
  15. eodag/plugins/apis/usgs.py +16 -15
  16. eodag/plugins/authentication/aws_auth.py +16 -13
  17. eodag/plugins/authentication/base.py +5 -3
  18. eodag/plugins/authentication/header.py +3 -3
  19. eodag/plugins/authentication/keycloak.py +4 -4
  20. eodag/plugins/authentication/oauth.py +7 -3
  21. eodag/plugins/authentication/openid_connect.py +16 -16
  22. eodag/plugins/authentication/sas_auth.py +4 -4
  23. eodag/plugins/authentication/token.py +41 -10
  24. eodag/plugins/authentication/token_exchange.py +1 -1
  25. eodag/plugins/base.py +4 -4
  26. eodag/plugins/crunch/base.py +4 -4
  27. eodag/plugins/crunch/filter_date.py +4 -4
  28. eodag/plugins/crunch/filter_latest_intersect.py +6 -6
  29. eodag/plugins/crunch/filter_latest_tpl_name.py +7 -7
  30. eodag/plugins/crunch/filter_overlap.py +4 -4
  31. eodag/plugins/crunch/filter_property.py +6 -7
  32. eodag/plugins/download/aws.py +58 -78
  33. eodag/plugins/download/base.py +38 -56
  34. eodag/plugins/download/creodias_s3.py +29 -0
  35. eodag/plugins/download/http.py +173 -183
  36. eodag/plugins/download/s3rest.py +10 -11
  37. eodag/plugins/manager.py +10 -20
  38. eodag/plugins/search/__init__.py +6 -5
  39. eodag/plugins/search/base.py +87 -44
  40. eodag/plugins/search/build_search_result.py +1067 -329
  41. eodag/plugins/search/cop_marine.py +22 -12
  42. eodag/plugins/search/creodias_s3.py +9 -73
  43. eodag/plugins/search/csw.py +11 -11
  44. eodag/plugins/search/data_request_search.py +16 -15
  45. eodag/plugins/search/qssearch.py +103 -187
  46. eodag/plugins/search/stac_list_assets.py +85 -0
  47. eodag/plugins/search/static_stac_search.py +3 -3
  48. eodag/resources/ext_product_types.json +1 -1
  49. eodag/resources/product_types.yml +663 -304
  50. eodag/resources/providers.yml +823 -1749
  51. eodag/resources/stac_api.yml +2 -2
  52. eodag/resources/user_conf_template.yml +11 -0
  53. eodag/rest/cache.py +2 -2
  54. eodag/rest/config.py +3 -3
  55. eodag/rest/core.py +112 -82
  56. eodag/rest/errors.py +5 -5
  57. eodag/rest/server.py +33 -14
  58. eodag/rest/stac.py +40 -38
  59. eodag/rest/types/collections_search.py +3 -3
  60. eodag/rest/types/eodag_search.py +29 -23
  61. eodag/rest/types/queryables.py +15 -16
  62. eodag/rest/types/stac_search.py +15 -25
  63. eodag/rest/utils/__init__.py +14 -21
  64. eodag/rest/utils/cql_evaluate.py +6 -6
  65. eodag/rest/utils/rfc3339.py +2 -2
  66. eodag/types/__init__.py +75 -28
  67. eodag/types/bbox.py +2 -2
  68. eodag/types/download_args.py +3 -3
  69. eodag/types/queryables.py +183 -72
  70. eodag/types/search_args.py +4 -4
  71. eodag/types/whoosh.py +127 -3
  72. eodag/utils/__init__.py +152 -50
  73. eodag/utils/exceptions.py +28 -21
  74. eodag/utils/import_system.py +2 -2
  75. eodag/utils/repr.py +65 -6
  76. eodag/utils/requests.py +13 -13
  77. eodag/utils/rest.py +2 -2
  78. eodag/utils/s3.py +208 -0
  79. eodag/utils/stac_reader.py +10 -10
  80. {eodag-3.0.1.dist-info → eodag-3.1.0b2.dist-info}/METADATA +77 -76
  81. eodag-3.1.0b2.dist-info/RECORD +113 -0
  82. {eodag-3.0.1.dist-info → eodag-3.1.0b2.dist-info}/WHEEL +1 -1
  83. {eodag-3.0.1.dist-info → eodag-3.1.0b2.dist-info}/entry_points.txt +4 -2
  84. eodag/utils/constraints.py +0 -244
  85. eodag-3.0.1.dist-info/RECORD +0 -109
  86. {eodag-3.0.1.dist-info → eodag-3.1.0b2.dist-info}/LICENSE +0 -0
  87. {eodag-3.0.1.dist-info → eodag-3.1.0b2.dist-info}/top_level.txt +0 -0
eodag/api/core.py CHANGED
@@ -24,27 +24,15 @@ import re
24
24
  import shutil
25
25
  import tempfile
26
26
  from operator import itemgetter
27
- from typing import (
28
- TYPE_CHECKING,
29
- Annotated,
30
- Any,
31
- Dict,
32
- Iterator,
33
- List,
34
- Optional,
35
- Set,
36
- Tuple,
37
- Union,
38
- )
27
+ from typing import TYPE_CHECKING, Any, Iterator, Optional, Union
39
28
 
40
29
  import geojson
41
30
  import pkg_resources
42
31
  import yaml.parser
43
32
  from pkg_resources import resource_filename
44
- from pydantic.fields import FieldInfo
45
33
  from whoosh import analysis, fields
46
34
  from whoosh.fields import Schema
47
- from whoosh.index import create_in, exists_in, open_dir
35
+ from whoosh.index import exists_in, open_dir
48
36
  from whoosh.qparser import QueryParser
49
37
 
50
38
  from eodag.api.product.metadata_mapping import (
@@ -69,11 +57,11 @@ from eodag.config import (
69
57
  )
70
58
  from eodag.plugins.manager import PluginManager
71
59
  from eodag.plugins.search import PreparedSearch
72
- from eodag.plugins.search.build_search_result import BuildPostSearchResult
60
+ from eodag.plugins.search.build_search_result import MeteoblueSearch
73
61
  from eodag.plugins.search.qssearch import PostJsonSearch
74
62
  from eodag.types import model_fields_to_annotated
75
- from eodag.types.queryables import CommonQueryables
76
- from eodag.types.whoosh import EODAGQueryParser
63
+ from eodag.types.queryables import CommonQueryables, QueryablesDict
64
+ from eodag.types.whoosh import EODAGQueryParser, create_in
77
65
  from eodag.utils import (
78
66
  DEFAULT_DOWNLOAD_TIMEOUT,
79
67
  DEFAULT_DOWNLOAD_WAIT,
@@ -84,7 +72,6 @@ from eodag.utils import (
84
72
  HTTP_REQ_TIMEOUT,
85
73
  MockResponse,
86
74
  _deprecated,
87
- copy_deepcopy,
88
75
  get_geometry_from_various,
89
76
  makedirs,
90
77
  obj_md5sum,
@@ -93,6 +80,7 @@ from eodag.utils import (
93
80
  uri_to_path,
94
81
  )
95
82
  from eodag.utils.exceptions import (
83
+ AuthenticationError,
96
84
  EodagError,
97
85
  NoMatchingProductType,
98
86
  PluginImplementationError,
@@ -197,7 +185,7 @@ class EODataAccessGateway:
197
185
  self._plugins_manager.rebuild(self.providers_config)
198
186
 
199
187
  # store pruned providers configs
200
- self._pruned_providers_config: Dict[str, Any] = {}
188
+ self._pruned_providers_config: dict[str, Any] = {}
201
189
  # filter out providers needing auth that have no credentials set
202
190
  self._prune_providers_list()
203
191
 
@@ -219,9 +207,10 @@ class EODataAccessGateway:
219
207
  "eodag",
220
208
  os.path.join("resources", "locations_conf_template.yml"),
221
209
  )
222
- with open(locations_conf_template) as infile, open(
223
- locations_conf_path, "w"
224
- ) as outfile:
210
+ with (
211
+ open(locations_conf_template) as infile,
212
+ open(locations_conf_path, "w") as outfile,
213
+ ):
225
214
  # The template contains paths in the form of:
226
215
  # /path/to/locations/file.shp
227
216
  path_template = "/path/to/locations/"
@@ -317,13 +306,18 @@ class EODataAccessGateway:
317
306
  product_type, **{"md5": self.product_types_config_md5}
318
307
  )
319
308
  # add to index
320
- ix_writer.add_document(
321
- **{
322
- k: v
323
- for k, v in versioned_product_type.items()
324
- if k in product_types_schema.names()
325
- }
326
- )
309
+ try:
310
+ ix_writer.add_document(
311
+ **{
312
+ k: v
313
+ for k, v in versioned_product_type.items()
314
+ if k in product_types_schema.names()
315
+ }
316
+ )
317
+ except TypeError as e:
318
+ logger.error(
319
+ f"Cannot write product type {product_type['ID']} into index. e={e} product_type={product_type}"
320
+ )
327
321
  ix_writer.commit()
328
322
 
329
323
  def set_preferred_provider(self, provider: str) -> None:
@@ -341,7 +335,7 @@ class EODataAccessGateway:
341
335
  new_priority = max_priority + 1
342
336
  self._plugins_manager.set_priority(provider, new_priority)
343
337
 
344
- def get_preferred_provider(self) -> Tuple[str, int]:
338
+ def get_preferred_provider(self) -> tuple[str, int]:
345
339
  """Get the provider currently set as the preferred one for searching
346
340
  products, along with its priority.
347
341
 
@@ -357,7 +351,7 @@ class EODataAccessGateway:
357
351
  def update_providers_config(
358
352
  self,
359
353
  yaml_conf: Optional[str] = None,
360
- dict_conf: Optional[Dict[str, Any]] = None,
354
+ dict_conf: Optional[dict[str, Any]] = None,
361
355
  ) -> None:
362
356
  """Update providers configuration with given input.
363
357
  Can be used to add a provider to existing configuration or update
@@ -403,12 +397,12 @@ class EODataAccessGateway:
403
397
  name: str,
404
398
  url: Optional[str] = None,
405
399
  priority: Optional[int] = None,
406
- search: Dict[str, Any] = {"type": "StacSearch"},
407
- products: Dict[str, Any] = {
400
+ search: dict[str, Any] = {"type": "StacSearch"},
401
+ products: dict[str, Any] = {
408
402
  GENERIC_PRODUCT_TYPE: {"productType": "{productType}"}
409
403
  },
410
- download: Dict[str, Any] = {"type": "HTTPDownload", "auth_error_code": 401},
411
- **kwargs: Dict[str, Any],
404
+ download: dict[str, Any] = {"type": "HTTPDownload", "auth_error_code": 401},
405
+ **kwargs: dict[str, Any],
412
406
  ):
413
407
  """Adds a new provider.
414
408
 
@@ -427,7 +421,7 @@ class EODataAccessGateway:
427
421
  :param download: Download :class:`~eodag.config.PluginConfig` mapping
428
422
  :param kwargs: Additional :class:`~eodag.config.ProviderConfig` mapping
429
423
  """
430
- conf_dict: Dict[str, Any] = {
424
+ conf_dict: dict[str, Any] = {
431
425
  name: {
432
426
  "url": url,
433
427
  "search": {"type": "StacSearch", **search},
@@ -571,7 +565,7 @@ class EODataAccessGateway:
571
565
  main_locations_config = locations_config[main_key]
572
566
 
573
567
  logger.info("Locations configuration loaded from %s" % locations_conf_path)
574
- self.locations_config: List[Dict[str, Any]] = main_locations_config
568
+ self.locations_config: list[dict[str, Any]] = main_locations_config
575
569
  else:
576
570
  logger.info(
577
571
  "Could not load locations configuration from %s" % locations_conf_path
@@ -580,7 +574,7 @@ class EODataAccessGateway:
580
574
 
581
575
  def list_product_types(
582
576
  self, provider: Optional[str] = None, fetch_providers: bool = True
583
- ) -> List[Dict[str, Any]]:
577
+ ) -> list[dict[str, Any]]:
584
578
  """Lists supported product types.
585
579
 
586
580
  :param provider: (optional) The name of a provider that must support the product
@@ -594,7 +588,7 @@ class EODataAccessGateway:
594
588
  # First, update product types list if possible
595
589
  self.fetch_product_types_list(provider=provider)
596
590
 
597
- product_types: List[Dict[str, Any]] = []
591
+ product_types: list[dict[str, Any]] = []
598
592
 
599
593
  providers_configs = (
600
594
  list(self.providers_config.values())
@@ -650,7 +644,7 @@ class EODataAccessGateway:
650
644
  providers_to_fetch = [provider]
651
645
 
652
646
  # providers discovery confs that are fetchable
653
- providers_discovery_configs_fetchable: Dict[str, Any] = {}
647
+ providers_discovery_configs_fetchable: dict[str, Any] = {}
654
648
  # check if any provider has not already been fetched for product types
655
649
  already_fetched = True
656
650
  for provider_to_fetch in providers_to_fetch:
@@ -773,7 +767,7 @@ class EODataAccessGateway:
773
767
 
774
768
  def discover_product_types(
775
769
  self, provider: Optional[str] = None
776
- ) -> Optional[Dict[str, Any]]:
770
+ ) -> Optional[dict[str, Any]]:
777
771
  """Fetch providers for product types
778
772
 
779
773
  :param provider: The name of a provider or provider-group to fetch. Defaults to
@@ -793,7 +787,7 @@ class EODataAccessGateway:
793
787
  raise UnsupportedProvider(
794
788
  f"The requested provider is not (yet) supported: {provider}"
795
789
  )
796
- ext_product_types_conf: Dict[str, Any] = {}
790
+ ext_product_types_conf: dict[str, Any] = {}
797
791
  providers_to_fetch = [
798
792
  p
799
793
  for p in (
@@ -806,7 +800,7 @@ class EODataAccessGateway:
806
800
  else self.available_providers()
807
801
  )
808
802
  ]
809
- kwargs: Dict[str, Any] = {}
803
+ kwargs: dict[str, Any] = {}
810
804
  for provider in providers_to_fetch:
811
805
  if hasattr(self.providers_config[provider], "search"):
812
806
  search_plugin_config = self.providers_config[provider].search
@@ -847,7 +841,7 @@ class EODataAccessGateway:
847
841
  return sort_dict(ext_product_types_conf)
848
842
 
849
843
  def update_product_types_list(
850
- self, ext_product_types_conf: Dict[str, Optional[Dict[str, Dict[str, Any]]]]
844
+ self, ext_product_types_conf: dict[str, Optional[dict[str, dict[str, Any]]]]
851
845
  ) -> None:
852
846
  """Update eodag product types list
853
847
 
@@ -875,7 +869,7 @@ class EODataAccessGateway:
875
869
  provider,
876
870
  )
877
871
  continue
878
- new_product_types: List[str] = []
872
+ new_product_types: list[str] = []
879
873
  for (
880
874
  new_product_type,
881
875
  new_product_type_conf,
@@ -938,7 +932,7 @@ class EODataAccessGateway:
938
932
 
939
933
  def available_providers(
940
934
  self, product_type: Optional[str] = None, by_group: bool = False
941
- ) -> List[str]:
935
+ ) -> list[str]:
942
936
  """Gives the sorted list of the available providers or groups
943
937
 
944
938
  The providers or groups are sorted first by their priority level in descending order,
@@ -965,7 +959,7 @@ class EODataAccessGateway:
965
959
 
966
960
  # If by_group is True, keep only the highest priority for each group
967
961
  if by_group:
968
- group_priority: Dict[str, int] = {}
962
+ group_priority: dict[str, int] = {}
969
963
  for name, priority in providers:
970
964
  if name not in group_priority or priority > group_priority[name]:
971
965
  group_priority[name] = priority
@@ -1032,7 +1026,7 @@ class EODataAccessGateway:
1032
1026
  missionStartDate: Optional[str] = None,
1033
1027
  missionEndDate: Optional[str] = None,
1034
1028
  **kwargs: Any,
1035
- ) -> List[str]:
1029
+ ) -> list[str]:
1036
1030
  """
1037
1031
  Find EODAG product type IDs that best match a set of search parameters.
1038
1032
 
@@ -1090,7 +1084,7 @@ class EODataAccessGateway:
1090
1084
  query = p.parse(text)
1091
1085
  results = searcher.search(query, limit=None)
1092
1086
 
1093
- guesses: List[Dict[str, str]] = [dict(r) for r in results or []]
1087
+ guesses: list[dict[str, str]] = [dict(r) for r in results or []]
1094
1088
 
1095
1089
  # datetime filtering
1096
1090
  if missionStartDate or missionEndDate:
@@ -1131,8 +1125,8 @@ class EODataAccessGateway:
1131
1125
  raise_errors: bool = False,
1132
1126
  start: Optional[str] = None,
1133
1127
  end: Optional[str] = None,
1134
- geom: Optional[Union[str, Dict[str, float], BaseGeometry]] = None,
1135
- locations: Optional[Dict[str, str]] = None,
1128
+ geom: Optional[Union[str, dict[str, float], BaseGeometry]] = None,
1129
+ locations: Optional[dict[str, str]] = None,
1136
1130
  provider: Optional[str] = None,
1137
1131
  count: bool = False,
1138
1132
  **kwargs: Any,
@@ -1211,7 +1205,7 @@ class EODataAccessGateway:
1211
1205
  items_per_page=items_per_page,
1212
1206
  )
1213
1207
 
1214
- errors: List[Tuple[str, Exception]] = []
1208
+ errors: list[tuple[str, Exception]] = []
1215
1209
  # Loop over available providers and return the first non-empty results
1216
1210
  for i, search_plugin in enumerate(search_plugins):
1217
1211
  search_plugin.clear()
@@ -1240,8 +1234,8 @@ class EODataAccessGateway:
1240
1234
  items_per_page: int = DEFAULT_ITEMS_PER_PAGE,
1241
1235
  start: Optional[str] = None,
1242
1236
  end: Optional[str] = None,
1243
- geom: Optional[Union[str, Dict[str, float], BaseGeometry]] = None,
1244
- locations: Optional[Dict[str, str]] = None,
1237
+ geom: Optional[Union[str, dict[str, float], BaseGeometry]] = None,
1238
+ locations: Optional[dict[str, str]] = None,
1245
1239
  **kwargs: Any,
1246
1240
  ) -> Iterator[SearchResult]:
1247
1241
  """Iterate over the pages of a products search.
@@ -1417,8 +1411,8 @@ class EODataAccessGateway:
1417
1411
  items_per_page: Optional[int] = None,
1418
1412
  start: Optional[str] = None,
1419
1413
  end: Optional[str] = None,
1420
- geom: Optional[Union[str, Dict[str, float], BaseGeometry]] = None,
1421
- locations: Optional[Dict[str, str]] = None,
1414
+ geom: Optional[Union[str, dict[str, float], BaseGeometry]] = None,
1415
+ locations: Optional[dict[str, str]] = None,
1422
1416
  **kwargs: Any,
1423
1417
  ) -> SearchResult:
1424
1418
  """Search and return all the products matching the search criteria.
@@ -1639,7 +1633,7 @@ class EODataAccessGateway:
1639
1633
  if not getattr(plugin.config, "discover_product_types", {}).get("fetch_url"):
1640
1634
  return None
1641
1635
 
1642
- kwargs: Dict[str, Any] = {"productType": product_type}
1636
+ kwargs: dict[str, Any] = {"productType": product_type}
1643
1637
 
1644
1638
  # append auth if needed
1645
1639
  if getattr(plugin.config, "need_auth", False):
@@ -1657,11 +1651,11 @@ class EODataAccessGateway:
1657
1651
  self,
1658
1652
  start: Optional[str] = None,
1659
1653
  end: Optional[str] = None,
1660
- geom: Optional[Union[str, Dict[str, float], BaseGeometry]] = None,
1661
- locations: Optional[Dict[str, str]] = None,
1654
+ geom: Optional[Union[str, dict[str, float], BaseGeometry]] = None,
1655
+ locations: Optional[dict[str, str]] = None,
1662
1656
  provider: Optional[str] = None,
1663
1657
  **kwargs: Any,
1664
- ) -> Tuple[List[Union[Search, Api]], Dict[str, Any]]:
1658
+ ) -> tuple[list[Union[Search, Api]], dict[str, Any]]:
1665
1659
  """Internal method to prepare the search kwargs and get the search plugins.
1666
1660
 
1667
1661
  Product query:
@@ -1769,16 +1763,16 @@ class EODataAccessGateway:
1769
1763
 
1770
1764
  preferred_provider = self.get_preferred_provider()[0]
1771
1765
 
1772
- search_plugins: List[Union[Search, Api]] = []
1766
+ search_plugins: list[Union[Search, Api]] = []
1773
1767
  for plugin in self._plugins_manager.get_search_plugins(
1774
1768
  product_type=product_type, provider=provider
1775
1769
  ):
1776
- # exclude BuildPostSearchResult plugins from search fallback for unknow product_type
1770
+ # exclude MeteoblueSearch plugins from search fallback for unknown product_type
1777
1771
  if (
1778
1772
  provider != plugin.provider
1779
1773
  and preferred_provider != plugin.provider
1780
1774
  and product_type not in self.product_types_config
1781
- and isinstance(plugin, BuildPostSearchResult)
1775
+ and isinstance(plugin, MeteoblueSearch)
1782
1776
  ):
1783
1777
  continue
1784
1778
  search_plugins.append(plugin)
@@ -1801,27 +1795,7 @@ class EODataAccessGateway:
1801
1795
  # Add product_types_config to plugin config. This dict contains product
1802
1796
  # type metadata that will also be stored in each product's properties.
1803
1797
  for search_plugin in search_plugins:
1804
- try:
1805
- search_plugin.config.product_type_config = dict(
1806
- [
1807
- p
1808
- for p in self.list_product_types(
1809
- search_plugin.provider, fetch_providers=False
1810
- )
1811
- if p["_id"] == product_type
1812
- ][0],
1813
- **{"productType": product_type},
1814
- )
1815
- # If the product isn't in the catalog, it's a generic product type.
1816
- except IndexError:
1817
- # Construct the GENERIC_PRODUCT_TYPE metadata
1818
- search_plugin.config.product_type_config = dict(
1819
- ID=GENERIC_PRODUCT_TYPE,
1820
- **self.product_types_config[GENERIC_PRODUCT_TYPE],
1821
- productType=product_type,
1822
- )
1823
- # Remove the ID since this is equal to productType.
1824
- search_plugin.config.product_type_config.pop("ID", None)
1798
+ self._attach_product_type_config(search_plugin, product_type)
1825
1799
 
1826
1800
  return search_plugins, kwargs
1827
1801
 
@@ -1859,10 +1833,10 @@ class EODataAccessGateway:
1859
1833
  max_items_per_page,
1860
1834
  )
1861
1835
 
1862
- results: List[EOProduct] = []
1836
+ results: list[EOProduct] = []
1863
1837
  total_results: Optional[int] = 0 if count else None
1864
1838
 
1865
- errors: List[Tuple[str, Exception]] = []
1839
+ errors: list[tuple[str, Exception]] = []
1866
1840
 
1867
1841
  try:
1868
1842
  prep = PreparedSearch(count=count)
@@ -2010,7 +1984,7 @@ class EODataAccessGateway:
2010
1984
  return results
2011
1985
 
2012
1986
  @staticmethod
2013
- def group_by_extent(searches: List[SearchResult]) -> List[SearchResult]:
1987
+ def group_by_extent(searches: list[SearchResult]) -> list[SearchResult]:
2014
1988
  """Combines multiple SearchResults and return a list of SearchResults grouped
2015
1989
  by extent (i.e. bounding box).
2016
1990
 
@@ -2019,7 +1993,7 @@ class EODataAccessGateway:
2019
1993
  """
2020
1994
  # Dict with extents as keys, each extent being defined by a str
2021
1995
  # "{minx}{miny}{maxx}{maxy}" (each float rounded to 2 dec).
2022
- products_grouped_by_extent: Dict[str, Any] = {}
1996
+ products_grouped_by_extent: dict[str, Any] = {}
2023
1997
 
2024
1998
  for search in searches:
2025
1999
  for product in search:
@@ -2038,10 +2012,10 @@ class EODataAccessGateway:
2038
2012
  search_result: SearchResult,
2039
2013
  downloaded_callback: Optional[DownloadedCallback] = None,
2040
2014
  progress_callback: Optional[ProgressCallback] = None,
2041
- wait: int = DEFAULT_DOWNLOAD_WAIT,
2042
- timeout: int = DEFAULT_DOWNLOAD_TIMEOUT,
2015
+ wait: float = DEFAULT_DOWNLOAD_WAIT,
2016
+ timeout: float = DEFAULT_DOWNLOAD_TIMEOUT,
2043
2017
  **kwargs: Unpack[DownloadConf],
2044
- ) -> List[str]:
2018
+ ) -> list[str]:
2045
2019
  """Download all products resulting from a search.
2046
2020
 
2047
2021
  :param search_result: A collection of EO products resulting from a search
@@ -2201,8 +2175,8 @@ class EODataAccessGateway:
2201
2175
  self,
2202
2176
  product: EOProduct,
2203
2177
  progress_callback: Optional[ProgressCallback] = None,
2204
- wait: int = DEFAULT_DOWNLOAD_WAIT,
2205
- timeout: int = DEFAULT_DOWNLOAD_TIMEOUT,
2178
+ wait: float = DEFAULT_DOWNLOAD_WAIT,
2179
+ timeout: float = DEFAULT_DOWNLOAD_TIMEOUT,
2206
2180
  **kwargs: Unpack[DownloadConf],
2207
2181
  ) -> str:
2208
2182
  """Download a single product.
@@ -2280,71 +2254,107 @@ class EODataAccessGateway:
2280
2254
  return self._plugins_manager.get_crunch_plugin(name, **plugin_conf)
2281
2255
 
2282
2256
  def list_queryables(
2283
- self, provider: Optional[str] = None, **kwargs: Any
2284
- ) -> Dict[str, Annotated[Any, FieldInfo]]:
2257
+ self,
2258
+ provider: Optional[str] = None,
2259
+ fetch_providers: bool = True,
2260
+ **kwargs: Any,
2261
+ ) -> QueryablesDict:
2285
2262
  """Fetch the queryable properties for a given product type and/or provider.
2286
2263
 
2287
2264
  :param provider: (optional) The provider.
2265
+ :param fetch_providers: If new product types should be fetched from the providers; default: True
2288
2266
  :param kwargs: additional filters for queryables (`productType` or other search
2289
2267
  arguments)
2290
2268
 
2291
2269
  :raises UnsupportedProductType: If the specified product type is not available for the
2292
2270
  provider.
2293
2271
 
2294
- :returns: A dict containing the EODAG queryable properties, associating
2295
- parameters to their annotated type
2272
+ :returns: A :class:`~eodag.api.product.queryables.QuerybalesDict` containing the EODAG queryable
2273
+ properties, associating parameters to their annotated type, and a additional_properties attribute
2296
2274
  """
2297
- available_product_types = [
2275
+ # only fetch providers if product type is not found
2276
+ available_product_types: list[str] = [
2298
2277
  pt["ID"]
2299
2278
  for pt in self.list_product_types(provider=provider, fetch_providers=False)
2300
2279
  ]
2301
- product_type = kwargs.get("productType")
2280
+ product_type: Optional[str] = kwargs.get("productType")
2281
+ pt_alias: Optional[str] = product_type
2302
2282
 
2303
2283
  if product_type:
2284
+ if product_type not in available_product_types:
2285
+ if fetch_providers:
2286
+ # fetch providers and try again
2287
+ available_product_types = [
2288
+ pt["ID"]
2289
+ for pt in self.list_product_types(
2290
+ provider=provider, fetch_providers=True
2291
+ )
2292
+ ]
2293
+ raise UnsupportedProductType(f"{product_type} is not available.")
2304
2294
  try:
2305
2295
  kwargs["productType"] = product_type = self.get_product_type_from_alias(
2306
2296
  product_type
2307
2297
  )
2308
2298
  except NoMatchingProductType as e:
2309
- raise UnsupportedProductType(f"{product_type} is not available") from e
2310
-
2311
- if product_type and product_type not in available_product_types:
2312
- self.fetch_product_types_list()
2299
+ raise UnsupportedProductType(f"{product_type} is not available.") from e
2313
2300
 
2314
2301
  if not provider and not product_type:
2315
- return model_fields_to_annotated(CommonQueryables.model_fields)
2302
+ return QueryablesDict(
2303
+ additional_properties=True,
2304
+ **model_fields_to_annotated(CommonQueryables.model_fields),
2305
+ )
2316
2306
 
2317
- providers_queryables: Dict[str, Dict[str, Annotated[Any, FieldInfo]]] = {}
2307
+ additional_properties = False
2308
+ additional_information = []
2309
+ queryable_properties: dict[str, Any] = {}
2318
2310
 
2319
2311
  for plugin in self._plugins_manager.get_search_plugins(product_type, provider):
2312
+ # attach product type config
2313
+ product_type_configs: dict[str, Any] = {}
2314
+ if product_type:
2315
+ self._attach_product_type_config(plugin, product_type)
2316
+ product_type_configs[product_type] = plugin.config.product_type_config
2317
+ else:
2318
+ for pt in available_product_types:
2319
+ self._attach_product_type_config(plugin, pt)
2320
+ product_type_configs[pt] = plugin.config.product_type_config
2321
+
2322
+ # authenticate if required
2320
2323
  if getattr(plugin.config, "need_auth", False) and (
2321
2324
  auth := self._plugins_manager.get_auth_plugin(plugin)
2322
2325
  ):
2323
- plugin.auth = auth.authenticate()
2324
- providers_queryables[plugin.provider] = plugin.list_queryables(
2325
- filters=kwargs, product_type=product_type
2326
- )
2327
-
2328
- queryable_keys: Set[str] = set.intersection( # type: ignore
2329
- *[set(q.keys()) for q in providers_queryables.values()]
2330
- )
2331
- queryables = {
2332
- k: v
2333
- for k, v in list(providers_queryables.values())[0].items()
2334
- if k in queryable_keys
2335
- }
2326
+ try:
2327
+ plugin.auth = auth.authenticate()
2328
+ except AuthenticationError:
2329
+ logger.debug(
2330
+ "queryables from provider %s could not be fetched due to an authentication error",
2331
+ plugin.provider,
2332
+ )
2336
2333
 
2337
- # always keep at least CommonQueryables
2338
- common_queryables = copy_deepcopy(CommonQueryables.model_fields)
2339
- for key, queryable in common_queryables.items():
2340
- if key in kwargs:
2341
- queryable.default = kwargs[key]
2334
+ plugin_queryables = plugin.list_queryables(
2335
+ kwargs,
2336
+ available_product_types,
2337
+ product_type_configs,
2338
+ product_type,
2339
+ pt_alias,
2340
+ )
2342
2341
 
2343
- queryables.update(model_fields_to_annotated(common_queryables))
2342
+ if plugin_queryables.additional_information:
2343
+ additional_information.append(
2344
+ f"{plugin.provider}: {plugin_queryables.additional_information}"
2345
+ )
2346
+ queryable_properties = {**plugin_queryables, **queryable_properties}
2347
+ additional_properties = (
2348
+ additional_properties or plugin_queryables.additional_properties
2349
+ )
2344
2350
 
2345
- return queryables
2351
+ return QueryablesDict(
2352
+ additional_properties=additional_properties,
2353
+ additional_information=" | ".join(additional_information),
2354
+ **queryable_properties,
2355
+ )
2346
2356
 
2347
- def available_sortables(self) -> Dict[str, Optional[ProviderSortables]]:
2357
+ def available_sortables(self) -> dict[str, Optional[ProviderSortables]]:
2348
2358
  """For each provider, gives its available sortable parameter(s) and its maximum
2349
2359
  number of them if it supports the sorting feature, otherwise gives None.
2350
2360
 
@@ -2352,7 +2362,7 @@ class EODataAccessGateway:
2352
2362
  its (their) maximum number as value(s).
2353
2363
  :raises: :class:`~eodag.utils.exceptions.UnsupportedProvider`
2354
2364
  """
2355
- sortables: Dict[str, Optional[ProviderSortables]] = {}
2365
+ sortables: dict[str, Optional[ProviderSortables]] = {}
2356
2366
  provider_search_plugins = self._plugins_manager.get_search_plugins()
2357
2367
  for provider_search_plugin in provider_search_plugins:
2358
2368
  provider = provider_search_plugin.provider
@@ -2375,3 +2385,30 @@ class EODataAccessGateway:
2375
2385
  ],
2376
2386
  }
2377
2387
  return sortables
2388
+
2389
+ def _attach_product_type_config(self, plugin: Search, product_type: str) -> None:
2390
+ """
2391
+ Attach product_types_config to plugin config. This dict contains product
2392
+ type metadata that will also be stored in each product's properties.
2393
+ """
2394
+ try:
2395
+ plugin.config.product_type_config = dict(
2396
+ [
2397
+ p
2398
+ for p in self.list_product_types(
2399
+ plugin.provider, fetch_providers=False
2400
+ )
2401
+ if p["_id"] == product_type
2402
+ ][0],
2403
+ **{"productType": product_type},
2404
+ )
2405
+ # If the product isn't in the catalog, it's a generic product type.
2406
+ except IndexError:
2407
+ # Construct the GENERIC_PRODUCT_TYPE metadata
2408
+ plugin.config.product_type_config = dict(
2409
+ ID=GENERIC_PRODUCT_TYPE,
2410
+ **self.product_types_config[GENERIC_PRODUCT_TYPE],
2411
+ productType=product_type,
2412
+ )
2413
+ # Remove the ID since this is equal to productType.
2414
+ plugin.config.product_type_config.pop("ID", None)
@@ -19,7 +19,7 @@ from __future__ import annotations
19
19
 
20
20
  import re
21
21
  from collections import UserDict
22
- from typing import TYPE_CHECKING, Any, Dict, List, Optional
22
+ from typing import TYPE_CHECKING, Any, Optional
23
23
 
24
24
  from eodag.utils.exceptions import NotAvailableError
25
25
  from eodag.utils.repr import dict_to_html_table
@@ -45,10 +45,10 @@ class AssetsDict(UserDict):
45
45
  self.product = product
46
46
  super(AssetsDict, self).__init__(*args, **kwargs)
47
47
 
48
- def __setitem__(self, key: str, value: Dict[str, Any]) -> None:
48
+ def __setitem__(self, key: str, value: dict[str, Any]) -> None:
49
49
  super().__setitem__(key, Asset(self.product, key, value))
50
50
 
51
- def as_dict(self) -> Dict[str, Any]:
51
+ def as_dict(self) -> dict[str, Any]:
52
52
  """Builds a representation of AssetsDict to enable its serialization
53
53
 
54
54
  :returns: The representation of a :class:`~eodag.api.product._assets.AssetsDict`
@@ -56,7 +56,7 @@ class AssetsDict(UserDict):
56
56
  """
57
57
  return {k: v.as_dict() for k, v in self.data.items()}
58
58
 
59
- def get_values(self, asset_filter: str = "") -> List[Asset]:
59
+ def get_values(self, asset_filter: str = "") -> list[Asset]:
60
60
  """
61
61
  retrieves the assets matching the given filter
62
62
 
@@ -98,12 +98,12 @@ class AssetsDict(UserDict):
98
98
  <details><summary style='color: grey;'>
99
99
  <span style='color: black'>'{k}'</span>:&ensp;
100
100
  {{
101
- {"'roles': '<span style='color: black'>"+str(v['roles'])+"</span>',&ensp;"
102
- if v.get("roles") else ""}
103
- {"'type': '"+str(v['type'])+"',&ensp;"
104
- if v.get("type") else ""}
105
- {"'title': '<span style='color: black'>"+str(v['title'])+"</span>',&ensp;"
106
- if v.get("title") else ""}
101
+ {"'roles': '<span style='color: black'>" + str(v['roles']) + "</span>',&ensp;"
102
+ if v.get("roles") else ""}
103
+ {"'type': '" + str(v['type']) + "',&ensp;"
104
+ if v.get("type") else ""}
105
+ {"'title': '<span style='color: black'>" + str(v['title']) + "</span>',&ensp;"
106
+ if v.get("title") else ""}
107
107
  ...
108
108
  }}
109
109
  </summary>
@@ -138,7 +138,7 @@ class Asset(UserDict):
138
138
  self.key = key
139
139
  super(Asset, self).__init__(*args, **kwargs)
140
140
 
141
- def as_dict(self) -> Dict[str, Any]:
141
+ def as_dict(self) -> dict[str, Any]:
142
142
  """Builds a representation of Asset to enable its serialization
143
143
 
144
144
  :returns: The representation of a :class:`~eodag.api.product._assets.Asset` as a