eodag 3.0.0b3__py3-none-any.whl → 3.0.1__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (71) hide show
  1. eodag/api/core.py +189 -125
  2. eodag/api/product/metadata_mapping.py +12 -3
  3. eodag/api/search_result.py +29 -3
  4. eodag/cli.py +35 -19
  5. eodag/config.py +412 -116
  6. eodag/plugins/apis/base.py +10 -4
  7. eodag/plugins/apis/ecmwf.py +14 -4
  8. eodag/plugins/apis/usgs.py +25 -2
  9. eodag/plugins/authentication/aws_auth.py +14 -5
  10. eodag/plugins/authentication/base.py +10 -1
  11. eodag/plugins/authentication/generic.py +14 -3
  12. eodag/plugins/authentication/header.py +12 -4
  13. eodag/plugins/authentication/keycloak.py +41 -22
  14. eodag/plugins/authentication/oauth.py +11 -1
  15. eodag/plugins/authentication/openid_connect.py +178 -163
  16. eodag/plugins/authentication/qsauth.py +12 -4
  17. eodag/plugins/authentication/sas_auth.py +19 -2
  18. eodag/plugins/authentication/token.py +57 -10
  19. eodag/plugins/authentication/token_exchange.py +19 -19
  20. eodag/plugins/crunch/base.py +4 -1
  21. eodag/plugins/crunch/filter_date.py +5 -2
  22. eodag/plugins/crunch/filter_latest_intersect.py +5 -4
  23. eodag/plugins/crunch/filter_latest_tpl_name.py +1 -1
  24. eodag/plugins/crunch/filter_overlap.py +5 -7
  25. eodag/plugins/crunch/filter_property.py +4 -3
  26. eodag/plugins/download/aws.py +39 -22
  27. eodag/plugins/download/base.py +11 -11
  28. eodag/plugins/download/creodias_s3.py +11 -2
  29. eodag/plugins/download/http.py +86 -52
  30. eodag/plugins/download/s3rest.py +20 -18
  31. eodag/plugins/manager.py +168 -23
  32. eodag/plugins/search/base.py +33 -14
  33. eodag/plugins/search/build_search_result.py +55 -51
  34. eodag/plugins/search/cop_marine.py +112 -29
  35. eodag/plugins/search/creodias_s3.py +20 -5
  36. eodag/plugins/search/csw.py +41 -1
  37. eodag/plugins/search/data_request_search.py +109 -9
  38. eodag/plugins/search/qssearch.py +532 -152
  39. eodag/plugins/search/static_stac_search.py +20 -21
  40. eodag/resources/ext_product_types.json +1 -1
  41. eodag/resources/product_types.yml +187 -56
  42. eodag/resources/providers.yml +1610 -1701
  43. eodag/resources/stac.yml +3 -163
  44. eodag/resources/user_conf_template.yml +112 -97
  45. eodag/rest/config.py +1 -2
  46. eodag/rest/constants.py +0 -1
  47. eodag/rest/core.py +61 -51
  48. eodag/rest/errors.py +181 -0
  49. eodag/rest/server.py +24 -325
  50. eodag/rest/stac.py +93 -544
  51. eodag/rest/types/eodag_search.py +13 -8
  52. eodag/rest/types/queryables.py +1 -2
  53. eodag/rest/types/stac_search.py +11 -2
  54. eodag/types/__init__.py +15 -3
  55. eodag/types/download_args.py +1 -1
  56. eodag/types/queryables.py +1 -2
  57. eodag/types/search_args.py +3 -3
  58. eodag/utils/__init__.py +77 -57
  59. eodag/utils/exceptions.py +23 -9
  60. eodag/utils/logging.py +37 -77
  61. eodag/utils/requests.py +1 -3
  62. eodag/utils/stac_reader.py +1 -1
  63. {eodag-3.0.0b3.dist-info → eodag-3.0.1.dist-info}/METADATA +11 -12
  64. eodag-3.0.1.dist-info/RECORD +109 -0
  65. {eodag-3.0.0b3.dist-info → eodag-3.0.1.dist-info}/WHEEL +1 -1
  66. {eodag-3.0.0b3.dist-info → eodag-3.0.1.dist-info}/entry_points.txt +1 -0
  67. eodag/resources/constraints/climate-dt.json +0 -13
  68. eodag/resources/constraints/extremes-dt.json +0 -8
  69. eodag-3.0.0b3.dist-info/RECORD +0 -110
  70. {eodag-3.0.0b3.dist-info → eodag-3.0.1.dist-info}/LICENSE +0 -0
  71. {eodag-3.0.0b3.dist-info → eodag-3.0.1.dist-info}/top_level.txt +0 -0
eodag/api/core.py CHANGED
@@ -17,13 +17,25 @@
17
17
  # limitations under the License.
18
18
  from __future__ import annotations
19
19
 
20
+ import datetime
20
21
  import logging
21
22
  import os
22
23
  import re
23
24
  import shutil
24
25
  import tempfile
25
26
  from operator import itemgetter
26
- from typing import TYPE_CHECKING, Any, Dict, Iterator, List, Optional, Set, Tuple, Union
27
+ from typing import (
28
+ TYPE_CHECKING,
29
+ Annotated,
30
+ Any,
31
+ Dict,
32
+ Iterator,
33
+ List,
34
+ Optional,
35
+ Set,
36
+ Tuple,
37
+ Union,
38
+ )
27
39
 
28
40
  import geojson
29
41
  import pkg_resources
@@ -35,11 +47,16 @@ from whoosh.fields import Schema
35
47
  from whoosh.index import create_in, exists_in, open_dir
36
48
  from whoosh.qparser import QueryParser
37
49
 
38
- from eodag.api.product.metadata_mapping import mtd_cfg_as_conversion_and_querypath
50
+ from eodag.api.product.metadata_mapping import (
51
+ ONLINE_STATUS,
52
+ mtd_cfg_as_conversion_and_querypath,
53
+ )
39
54
  from eodag.api.search_result import SearchResult
40
55
  from eodag.config import (
56
+ PLUGINS_TOPICS_KEYS,
41
57
  PluginConfig,
42
58
  SimpleYamlProxyConfig,
59
+ credentials_in_auth,
43
60
  get_ext_product_types_conf,
44
61
  load_default_config,
45
62
  load_stac_provider_config,
@@ -48,10 +65,12 @@ from eodag.config import (
48
65
  override_config_from_file,
49
66
  override_config_from_mapping,
50
67
  provider_config_init,
68
+ share_credentials,
51
69
  )
52
70
  from eodag.plugins.manager import PluginManager
53
71
  from eodag.plugins.search import PreparedSearch
54
72
  from eodag.plugins.search.build_search_result import BuildPostSearchResult
73
+ from eodag.plugins.search.qssearch import PostJsonSearch
55
74
  from eodag.types import model_fields_to_annotated
56
75
  from eodag.types.queryables import CommonQueryables
57
76
  from eodag.types.whoosh import EODAGQueryParser
@@ -74,9 +93,7 @@ from eodag.utils import (
74
93
  uri_to_path,
75
94
  )
76
95
  from eodag.utils.exceptions import (
77
- AuthenticationError,
78
96
  EodagError,
79
- MisconfiguredError,
80
97
  NoMatchingProductType,
81
98
  PluginImplementationError,
82
99
  RequestError,
@@ -96,7 +113,7 @@ if TYPE_CHECKING:
96
113
  from eodag.plugins.search.base import Search
97
114
  from eodag.types import ProviderSortables
98
115
  from eodag.types.download_args import DownloadConf
99
- from eodag.utils import Annotated, DownloadedCallback, ProgressCallback, Unpack
116
+ from eodag.utils import DownloadedCallback, ProgressCallback, Unpack
100
117
 
101
118
  logger = logging.getLogger("eodag.core")
102
119
 
@@ -166,10 +183,15 @@ class EODataAccessGateway:
166
183
  # Second level override: From environment variables
167
184
  override_config_from_env(self.providers_config)
168
185
 
186
+ # share credentials between updated plugins confs
187
+ share_credentials(self.providers_config)
188
+
169
189
  # init updated providers conf
170
- stac_provider_config = load_stac_provider_config()
171
190
  for provider in self.providers_config.keys():
172
- provider_config_init(self.providers_config[provider], stac_provider_config)
191
+ provider_config_init(
192
+ self.providers_config[provider],
193
+ load_stac_provider_config(),
194
+ )
173
195
 
174
196
  # re-build _plugins_manager using up-to-date providers_config
175
197
  self._plugins_manager.rebuild(self.providers_config)
@@ -215,7 +237,6 @@ class EODataAccessGateway:
215
237
  os.path.join(self.conf_dir, "shp"),
216
238
  )
217
239
  self.set_locations_conf(locations_conf_path)
218
- self.search_errors: Set = set()
219
240
 
220
241
  def get_version(self) -> str:
221
242
  """Get eodag package version"""
@@ -333,14 +354,24 @@ class EODataAccessGateway:
333
354
  preferred, priority = max(providers_with_priority, key=itemgetter(1))
334
355
  return preferred, priority
335
356
 
336
- def update_providers_config(self, yaml_conf: str) -> None:
357
+ def update_providers_config(
358
+ self,
359
+ yaml_conf: Optional[str] = None,
360
+ dict_conf: Optional[Dict[str, Any]] = None,
361
+ ) -> None:
337
362
  """Update providers configuration with given input.
338
363
  Can be used to add a provider to existing configuration or update
339
364
  an existing one.
340
365
 
341
366
  :param yaml_conf: YAML formated provider configuration
367
+ :param dict_conf: provider configuration as dictionary in place of ``yaml_conf``
342
368
  """
343
- conf_update = yaml.safe_load(yaml_conf)
369
+ if dict_conf is not None:
370
+ conf_update = dict_conf
371
+ elif yaml_conf is not None:
372
+ conf_update = yaml.safe_load(yaml_conf)
373
+ else:
374
+ return None
344
375
 
345
376
  # restore the pruned configuration
346
377
  for provider in list(self._pruned_providers_config.keys()):
@@ -355,9 +386,14 @@ class EODataAccessGateway:
355
386
 
356
387
  override_config_from_mapping(self.providers_config, conf_update)
357
388
 
358
- stac_provider_config = load_stac_provider_config()
389
+ # share credentials between updated plugins confs
390
+ share_credentials(self.providers_config)
391
+
359
392
  for provider in conf_update.keys():
360
- provider_config_init(self.providers_config[provider], stac_provider_config)
393
+ provider_config_init(
394
+ self.providers_config[provider],
395
+ load_stac_provider_config(),
396
+ )
361
397
  setattr(self.providers_config[provider], "product_types_fetched", False)
362
398
  # re-create _plugins_manager using up-to-date providers_config
363
399
  self._plugins_manager.build_product_type_to_provider_config_map()
@@ -419,14 +455,11 @@ class EODataAccessGateway:
419
455
 
420
456
  # api plugin usage: remove unneeded search/download/auth plugin conf
421
457
  if conf_dict[name].get("api"):
422
- conf_dict[name].pop("search", None)
423
- conf_dict[name].pop("download", None)
424
- conf_dict[name].pop("auth", None)
458
+ for k in PLUGINS_TOPICS_KEYS:
459
+ if k != "api":
460
+ conf_dict[name].pop(k, None)
425
461
 
426
- override_config_from_mapping(self.providers_config, conf_dict)
427
- provider_config_init(self.providers_config[name], load_stac_provider_config())
428
- setattr(self.providers_config[name], "product_types_fetched", False)
429
- self._plugins_manager.build_product_type_to_provider_config_map()
462
+ self.update_providers_config(dict_conf=conf_dict)
430
463
 
431
464
  if priority is None:
432
465
  self.set_preferred_provider(name)
@@ -452,12 +485,7 @@ class EODataAccessGateway:
452
485
 
453
486
  # check authentication
454
487
  if hasattr(conf, "api") and getattr(conf.api, "need_auth", False):
455
- credentials_exist = any(
456
- [
457
- cred is not None
458
- for cred in getattr(conf.api, "credentials", {}).values()
459
- ]
460
- )
488
+ credentials_exist = credentials_in_auth(conf.api)
461
489
  if not credentials_exist:
462
490
  # credentials needed but not found
463
491
  self._pruned_providers_config[provider] = self.providers_config.pop(
@@ -469,7 +497,7 @@ class EODataAccessGateway:
469
497
  provider,
470
498
  )
471
499
  elif hasattr(conf, "search") and getattr(conf.search, "need_auth", False):
472
- if not hasattr(conf, "auth"):
500
+ if not hasattr(conf, "auth") and not hasattr(conf, "search_auth"):
473
501
  # credentials needed but no auth plugin was found
474
502
  self._pruned_providers_config[provider] = self.providers_config.pop(
475
503
  provider
@@ -480,11 +508,13 @@ class EODataAccessGateway:
480
508
  provider,
481
509
  )
482
510
  continue
483
- credentials_exist = any(
484
- [
485
- cred is not None
486
- for cred in getattr(conf.auth, "credentials", {}).values()
487
- ]
511
+ credentials_exist = (
512
+ hasattr(conf, "search_auth")
513
+ and credentials_in_auth(conf.search_auth)
514
+ ) or (
515
+ not hasattr(conf, "search_auth")
516
+ and hasattr(conf, "auth")
517
+ and credentials_in_auth(conf.auth)
488
518
  )
489
519
  if not credentials_exist:
490
520
  # credentials needed but not found
@@ -599,21 +629,32 @@ class EODataAccessGateway:
599
629
  def fetch_product_types_list(self, provider: Optional[str] = None) -> None:
600
630
  """Fetch product types list and update if needed
601
631
 
602
- :param provider: (optional) The name of a provider for which product types list
603
- should be updated. Defaults to all providers (None value).
632
+ :param provider: The name of a provider or provider-group for which product types
633
+ list should be updated. Defaults to all providers (None value).
604
634
  """
635
+ providers_to_fetch = list(self.providers_config.keys())
636
+ # check if some providers are grouped under a group name which is not a provider name
605
637
  if provider is not None and provider not in self.providers_config:
606
- return
638
+ providers_to_fetch = [
639
+ p
640
+ for p, pconf in self.providers_config.items()
641
+ if provider == getattr(pconf, "group", None)
642
+ ]
643
+ if providers_to_fetch:
644
+ logger.info(
645
+ f"Fetch product types for {provider} group: {', '.join(providers_to_fetch)}"
646
+ )
647
+ else:
648
+ return None
649
+ elif provider is not None:
650
+ providers_to_fetch = [provider]
607
651
 
608
652
  # providers discovery confs that are fetchable
609
653
  providers_discovery_configs_fetchable: Dict[str, Any] = {}
610
654
  # check if any provider has not already been fetched for product types
611
655
  already_fetched = True
612
- for provider_to_fetch, provider_config in (
613
- {provider: self.providers_config[provider]}.items()
614
- if provider
615
- else self.providers_config.items()
616
- ):
656
+ for provider_to_fetch in providers_to_fetch:
657
+ provider_config = self.providers_config[provider_to_fetch]
617
658
  # get discovery conf
618
659
  if hasattr(provider_config, "search"):
619
660
  provider_search_config = provider_config.search
@@ -735,11 +776,20 @@ class EODataAccessGateway:
735
776
  ) -> Optional[Dict[str, Any]]:
736
777
  """Fetch providers for product types
737
778
 
738
- :param provider: (optional) The name of a provider to fetch. Defaults to all
739
- providers (None value).
779
+ :param provider: The name of a provider or provider-group to fetch. Defaults to
780
+ all providers (None value).
740
781
  :returns: external product types configuration
741
782
  """
742
- if provider and provider not in self.providers_config:
783
+ grouped_providers = [
784
+ p
785
+ for p, provider_config in self.providers_config.items()
786
+ if provider == getattr(provider_config, "group", None)
787
+ ]
788
+ if provider and provider not in self.providers_config and grouped_providers:
789
+ logger.info(
790
+ f"Discover product types for {provider} group: {', '.join(grouped_providers)}"
791
+ )
792
+ elif provider and provider not in self.providers_config:
743
793
  raise UnsupportedProvider(
744
794
  f"The requested provider is not (yet) supported: {provider}"
745
795
  )
@@ -748,7 +798,9 @@ class EODataAccessGateway:
748
798
  p
749
799
  for p in (
750
800
  [
751
- provider,
801
+ p
802
+ for p in self.providers_config
803
+ if p in grouped_providers + [provider]
752
804
  ]
753
805
  if provider
754
806
  else self.available_providers()
@@ -762,7 +814,9 @@ class EODataAccessGateway:
762
814
  search_plugin_config = self.providers_config[provider].api
763
815
  else:
764
816
  return None
765
- if getattr(search_plugin_config, "discover_product_types", None):
817
+ if getattr(search_plugin_config, "discover_product_types", {}).get(
818
+ "fetch_url", None
819
+ ):
766
820
  search_plugin: Union[Search, Api] = next(
767
821
  self._plugins_manager.get_search_plugins(provider=provider)
768
822
  )
@@ -773,23 +827,15 @@ class EODataAccessGateway:
773
827
  continue
774
828
  # append auth to search plugin if needed
775
829
  if getattr(search_plugin.config, "need_auth", False):
776
- auth_plugin = self._plugins_manager.get_auth_plugin(
777
- search_plugin.provider
778
- )
779
- if auth_plugin and callable(
780
- getattr(auth_plugin, "authenticate", None)
830
+ if auth := self._plugins_manager.get_auth(
831
+ search_plugin.provider,
832
+ getattr(search_plugin.config, "api_endpoint", None),
833
+ search_plugin.config,
781
834
  ):
782
- try:
783
- kwargs["auth"] = auth_plugin.authenticate()
784
- except (AuthenticationError, MisconfiguredError) as e:
785
- logger.warning(
786
- f"Could not authenticate on {provider}: {str(e)}"
787
- )
788
- ext_product_types_conf[provider] = None
789
- continue
835
+ kwargs["auth"] = auth
790
836
  else:
791
- logger.warning(
792
- f"Could not authenticate on {provider} using {auth_plugin} plugin"
837
+ logger.debug(
838
+ f"Could not authenticate on {provider} for product types discovery"
793
839
  )
794
840
  ext_product_types_conf[provider] = None
795
841
  continue
@@ -815,7 +861,9 @@ class EODataAccessGateway:
815
861
  ) or getattr(self.providers_config[provider], "api", None)
816
862
  if search_plugin_config is None:
817
863
  continue
818
- if not hasattr(search_plugin_config, "discover_product_types"):
864
+ if not getattr(
865
+ search_plugin_config, "discover_product_types", {}
866
+ ).get("fetch_url", None):
819
867
  # conf has been updated and provider product types are no more discoverable
820
868
  continue
821
869
  provider_products_config = (
@@ -874,7 +922,7 @@ class EODataAccessGateway:
874
922
  new_product_types.append(new_product_type)
875
923
  if new_product_types:
876
924
  logger.debug(
877
- f"Added product types {str(new_product_types)} for {provider}"
925
+ f"Added {len(new_product_types)} product types for {provider}"
878
926
  )
879
927
 
880
928
  elif provider not in self.providers_config:
@@ -1046,20 +1094,28 @@ class EODataAccessGateway:
1046
1094
 
1047
1095
  # datetime filtering
1048
1096
  if missionStartDate or missionEndDate:
1097
+ min_aware = datetime.datetime.min.replace(tzinfo=datetime.timezone.utc)
1098
+ max_aware = datetime.datetime.max.replace(tzinfo=datetime.timezone.utc)
1049
1099
  guesses = [
1050
1100
  g
1051
1101
  for g in guesses
1052
1102
  if (
1053
- not missionEndDate
1054
- or g.get("missionStartDate")
1055
- and rfc3339_str_to_datetime(g["missionStartDate"])
1056
- <= rfc3339_str_to_datetime(missionEndDate)
1057
- )
1058
- and (
1059
- not missionStartDate
1060
- or g.get("missionEndDate")
1061
- and rfc3339_str_to_datetime(g["missionEndDate"])
1062
- >= rfc3339_str_to_datetime(missionStartDate)
1103
+ max(
1104
+ rfc3339_str_to_datetime(missionStartDate)
1105
+ if missionStartDate
1106
+ else min_aware,
1107
+ rfc3339_str_to_datetime(g["missionStartDate"])
1108
+ if g.get("missionStartDate")
1109
+ else min_aware,
1110
+ )
1111
+ <= min(
1112
+ rfc3339_str_to_datetime(missionEndDate)
1113
+ if missionEndDate
1114
+ else max_aware,
1115
+ rfc3339_str_to_datetime(g["missionEndDate"])
1116
+ if g.get("missionEndDate")
1117
+ else max_aware,
1118
+ )
1063
1119
  )
1064
1120
  ]
1065
1121
 
@@ -1155,7 +1211,7 @@ class EODataAccessGateway:
1155
1211
  items_per_page=items_per_page,
1156
1212
  )
1157
1213
 
1158
- self.search_errors = set()
1214
+ errors: List[Tuple[str, Exception]] = []
1159
1215
  # Loop over available providers and return the first non-empty results
1160
1216
  for i, search_plugin in enumerate(search_plugins):
1161
1217
  search_plugin.clear()
@@ -1165,17 +1221,19 @@ class EODataAccessGateway:
1165
1221
  raise_errors=raise_errors,
1166
1222
  **search_kwargs,
1167
1223
  )
1224
+ errors.extend(search_results.errors)
1168
1225
  if len(search_results) == 0 and i < len(search_plugins) - 1:
1169
1226
  logger.warning(
1170
1227
  f"No result could be obtained from provider {search_plugin.provider}, "
1171
1228
  "we will try to get the data from another provider",
1172
1229
  )
1173
1230
  elif len(search_results) > 0:
1231
+ search_results.errors = errors
1174
1232
  return search_results
1175
1233
 
1176
1234
  if i > 1:
1177
1235
  logger.error("No result could be obtained from any available provider")
1178
- return SearchResult([], 0) if count else SearchResult([])
1236
+ return SearchResult([], 0, errors) if count else SearchResult([], errors=errors)
1179
1237
 
1180
1238
  def search_iter_page(
1181
1239
  self,
@@ -1435,7 +1493,7 @@ class EODataAccessGateway:
1435
1493
  )
1436
1494
  or DEFAULT_MAX_ITEMS_PER_PAGE
1437
1495
  )
1438
- logger.debug(
1496
+ logger.info(
1439
1497
  "Searching for all the products with provider %s and a maximum of %s "
1440
1498
  "items per page.",
1441
1499
  search_plugin.provider,
@@ -1503,7 +1561,7 @@ class EODataAccessGateway:
1503
1561
  try:
1504
1562
  product_type = self.get_product_type_from_alias(product_type)
1505
1563
  except NoMatchingProductType:
1506
- logger.warning("product type %s not found", product_type)
1564
+ logger.debug("product type %s not found", product_type)
1507
1565
  get_search_plugins_kwargs = dict(provider=provider, product_type=product_type)
1508
1566
  search_plugins = self._plugins_manager.get_search_plugins(
1509
1567
  **get_search_plugins_kwargs
@@ -1526,7 +1584,7 @@ class EODataAccessGateway:
1526
1584
  "max_items_per_page", DEFAULT_MAX_ITEMS_PER_PAGE
1527
1585
  )
1528
1586
  kwargs.update(items_per_page=items_per_page)
1529
- if isinstance(plugin, BuildPostSearchResult):
1587
+ if isinstance(plugin, PostJsonSearch):
1530
1588
  kwargs.update(
1531
1589
  items_per_page=items_per_page,
1532
1590
  _dc_qs=_dc_qs,
@@ -1544,9 +1602,10 @@ class EODataAccessGateway:
1544
1602
  **kwargs,
1545
1603
  ):
1546
1604
  results.data.extend(page_results.data)
1547
- except Exception:
1605
+ except Exception as e:
1548
1606
  if kwargs.get("raise_errors"):
1549
1607
  raise
1608
+ logger.warning(e)
1550
1609
  continue
1551
1610
 
1552
1611
  # try using crunch to get unique result
@@ -1584,16 +1643,12 @@ class EODataAccessGateway:
1584
1643
 
1585
1644
  # append auth if needed
1586
1645
  if getattr(plugin.config, "need_auth", False):
1587
- auth_plugin = self._plugins_manager.get_auth_plugin(plugin.provider)
1588
- if auth_plugin and callable(getattr(auth_plugin, "authenticate", None)):
1589
- try:
1590
- kwargs["auth"] = auth_plugin.authenticate()
1591
- except (AuthenticationError, MisconfiguredError) as e:
1592
- logger.warning(f"Could not authenticate on {provider}: {str(e)}")
1593
- else:
1594
- logger.warning(
1595
- f"Could not authenticate on {provider} using {auth_plugin} plugin"
1596
- )
1646
+ if auth := self._plugins_manager.get_auth(
1647
+ plugin.provider,
1648
+ getattr(plugin.config, "api_endpoint", None),
1649
+ plugin.config,
1650
+ ):
1651
+ kwargs["auth"] = auth
1597
1652
 
1598
1653
  product_type_config = plugin.discover_product_types(**kwargs)
1599
1654
  self.update_product_types_list({provider: product_type_config})
@@ -1732,12 +1787,10 @@ class EODataAccessGateway:
1732
1787
  provider = preferred_provider
1733
1788
  providers = [plugin.provider for plugin in search_plugins]
1734
1789
  if provider not in providers:
1735
- logger.warning(
1736
- "Product type '%s' is not available with provider '%s'. "
1737
- "Searching it on provider '%s' instead.",
1790
+ logger.debug(
1791
+ "Product type '%s' is not available with preferred provider '%s'.",
1738
1792
  product_type,
1739
1793
  provider,
1740
- search_plugins[0].provider,
1741
1794
  )
1742
1795
  else:
1743
1796
  provider_plugin = list(
@@ -1745,11 +1798,6 @@ class EODataAccessGateway:
1745
1798
  )[0]
1746
1799
  search_plugins.remove(provider_plugin)
1747
1800
  search_plugins.insert(0, provider_plugin)
1748
- logger.info(
1749
- "Searching product type '%s' on provider: %s",
1750
- product_type,
1751
- search_plugins[0].provider,
1752
- )
1753
1801
  # Add product_types_config to plugin config. This dict contains product
1754
1802
  # type metadata that will also be stored in each product's properties.
1755
1803
  for search_plugin in search_plugins:
@@ -1793,6 +1841,7 @@ class EODataAccessGateway:
1793
1841
  :param kwargs: Some other criteria that will be used to do the search
1794
1842
  :returns: A collection of EO products matching the criteria
1795
1843
  """
1844
+ logger.info("Searching on provider %s", search_plugin.provider)
1796
1845
  max_items_per_page = getattr(search_plugin.config, "pagination", {}).get(
1797
1846
  "max_items_per_page", DEFAULT_MAX_ITEMS_PER_PAGE
1798
1847
  )
@@ -1810,19 +1859,23 @@ class EODataAccessGateway:
1810
1859
  max_items_per_page,
1811
1860
  )
1812
1861
 
1813
- need_auth = getattr(search_plugin.config, "need_auth", False)
1814
- auth_plugin = self._plugins_manager.get_auth_plugin(search_plugin.provider)
1815
- can_authenticate = callable(getattr(auth_plugin, "authenticate", None))
1816
-
1817
1862
  results: List[EOProduct] = []
1818
1863
  total_results: Optional[int] = 0 if count else None
1819
1864
 
1865
+ errors: List[Tuple[str, Exception]] = []
1866
+
1820
1867
  try:
1821
1868
  prep = PreparedSearch(count=count)
1822
- if need_auth and auth_plugin and can_authenticate:
1823
- prep.auth = auth_plugin.authenticate()
1824
1869
 
1825
- prep.auth_plugin = auth_plugin
1870
+ # append auth if needed
1871
+ if getattr(search_plugin.config, "need_auth", False):
1872
+ if auth := self._plugins_manager.get_auth(
1873
+ search_plugin.provider,
1874
+ getattr(search_plugin.config, "api_endpoint", None),
1875
+ search_plugin.config,
1876
+ ):
1877
+ prep.auth = auth
1878
+
1826
1879
  prep.page = kwargs.pop("page", None)
1827
1880
  prep.items_per_page = kwargs.pop("items_per_page", None)
1828
1881
 
@@ -1876,12 +1929,31 @@ class EODataAccessGateway:
1876
1929
  eo_product.product_type
1877
1930
  )
1878
1931
  except NoMatchingProductType:
1879
- logger.warning("product type %s not found", eo_product.product_type)
1932
+ logger.debug("product type %s not found", eo_product.product_type)
1880
1933
 
1881
1934
  if eo_product.search_intersection is not None:
1882
1935
  download_plugin = self._plugins_manager.get_download_plugin(
1883
1936
  eo_product
1884
1937
  )
1938
+ if len(eo_product.assets) > 0:
1939
+ matching_url = next(iter(eo_product.assets.values()))["href"]
1940
+ elif eo_product.properties.get("storageStatus") != ONLINE_STATUS:
1941
+ matching_url = eo_product.properties.get(
1942
+ "orderLink"
1943
+ ) or eo_product.properties.get("downloadLink")
1944
+ else:
1945
+ matching_url = eo_product.properties.get("downloadLink")
1946
+
1947
+ try:
1948
+ auth_plugin = next(
1949
+ self._plugins_manager.get_auth_plugins(
1950
+ search_plugin.provider,
1951
+ matching_url=matching_url,
1952
+ matching_conf=download_plugin.config,
1953
+ )
1954
+ )
1955
+ except StopIteration:
1956
+ auth_plugin = None
1885
1957
  eo_product.register_downloader(download_plugin, auth_plugin)
1886
1958
 
1887
1959
  results.extend(res)
@@ -1911,13 +1983,6 @@ class EODataAccessGateway:
1911
1983
  "the total number of products matching the search criteria"
1912
1984
  )
1913
1985
  except Exception as e:
1914
- log_msg = f"No result from provider '{search_plugin.provider}' due to an error during search."
1915
- if not raise_errors:
1916
- log_msg += " Raise verbosity of log messages for details"
1917
- logger.info(log_msg)
1918
- # keep only the message from exception args
1919
- if len(e.args) > 1:
1920
- e.args = (e.args[0],)
1921
1986
  if raise_errors:
1922
1987
  # Raise the error, letting the application wrapping eodag know that
1923
1988
  # something went bad. This way it will be able to decide what to do next
@@ -1927,8 +1992,8 @@ class EODataAccessGateway:
1927
1992
  "Error while searching on provider %s (ignored):",
1928
1993
  search_plugin.provider,
1929
1994
  )
1930
- self.search_errors.add((search_plugin.provider, e))
1931
- return SearchResult(results, total_results)
1995
+ errors.append((search_plugin.provider, e))
1996
+ return SearchResult(results, total_results, errors)
1932
1997
 
1933
1998
  def crunch(self, results: SearchResult, **kwargs: Any) -> SearchResult:
1934
1999
  """Apply the filters given through the keyword arguments to the results
@@ -1982,7 +2047,7 @@ class EODataAccessGateway:
1982
2047
  :param search_result: A collection of EO products resulting from a search
1983
2048
  :param downloaded_callback: (optional) A method or a callable object which takes
1984
2049
  as parameter the ``product``. You can use the base class
1985
- :class:`~eodag.api.product.DownloadedCallback` and override
2050
+ :class:`~eodag.utils.DownloadedCallback` and override
1986
2051
  its ``__call__`` method. Will be called each time a product
1987
2052
  finishes downloading
1988
2053
  :param progress_callback: (optional) A method or a callable object
@@ -2061,12 +2126,12 @@ class EODataAccessGateway:
2061
2126
  products = self.deserialize(filename)
2062
2127
  for i, product in enumerate(products):
2063
2128
  if product.downloader is None:
2129
+ downloader = self._plugins_manager.get_download_plugin(product)
2064
2130
  auth = product.downloader_auth
2065
2131
  if auth is None:
2066
- auth = self._plugins_manager.get_auth_plugin(product.provider)
2067
- products[i].register_downloader(
2068
- self._plugins_manager.get_download_plugin(product), auth
2069
- )
2132
+ auth = self._plugins_manager.get_auth_plugin(downloader, product)
2133
+ products[i].register_downloader(downloader, auth)
2134
+
2070
2135
  return products
2071
2136
 
2072
2137
  @_deprecated(
@@ -2090,7 +2155,7 @@ class EODataAccessGateway:
2090
2155
 
2091
2156
  :param filename: A filename containing features encoded as a geojson
2092
2157
  :param recursive: (optional) Browse recursively in child nodes if True
2093
- :param max_connections: (optional) Maximum number of connections for HTTP requests
2158
+ :param max_connections: (optional) Maximum number of connections for concurrent HTTP requests
2094
2159
  :param provider: (optional) Data provider
2095
2160
  :param productType: (optional) Data product type
2096
2161
  :param timeout: (optional) Timeout in seconds for each internal HTTP request
@@ -2197,12 +2262,11 @@ class EODataAccessGateway:
2197
2262
 
2198
2263
  def _setup_downloader(self, product: EOProduct) -> None:
2199
2264
  if product.downloader is None:
2265
+ downloader = self._plugins_manager.get_download_plugin(product)
2200
2266
  auth = product.downloader_auth
2201
2267
  if auth is None:
2202
- auth = self._plugins_manager.get_auth_plugin(product.provider)
2203
- product.register_downloader(
2204
- self._plugins_manager.get_download_plugin(product), auth
2205
- )
2268
+ auth = self._plugins_manager.get_auth_plugin(downloader, product)
2269
+ product.register_downloader(downloader, auth)
2206
2270
 
2207
2271
  def get_cruncher(self, name: str, **options: Any) -> Crunch:
2208
2272
  """Build a crunch plugin from a configuration
@@ -2254,7 +2318,7 @@ class EODataAccessGateway:
2254
2318
 
2255
2319
  for plugin in self._plugins_manager.get_search_plugins(product_type, provider):
2256
2320
  if getattr(plugin.config, "need_auth", False) and (
2257
- auth := self._plugins_manager.get_auth_plugin(plugin.provider)
2321
+ auth := self._plugins_manager.get_auth_plugin(plugin)
2258
2322
  ):
2259
2323
  plugin.auth = auth.authenticate()
2260
2324
  providers_queryables[plugin.provider] = plugin.list_queryables(
@@ -2284,7 +2348,7 @@ class EODataAccessGateway:
2284
2348
  """For each provider, gives its available sortable parameter(s) and its maximum
2285
2349
  number of them if it supports the sorting feature, otherwise gives None.
2286
2350
 
2287
- :returns: A dictionnary with providers as keys and dictionnary of sortable parameter(s) and
2351
+ :returns: A dictionary with providers as keys and dictionary of sortable parameter(s) and
2288
2352
  its (their) maximum number as value(s).
2289
2353
  :raises: :class:`~eodag.utils.exceptions.UnsupportedProvider`
2290
2354
  """