eodag 3.0.0b2__py3-none-any.whl → 3.0.1__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (84) hide show
  1. eodag/__init__.py +6 -8
  2. eodag/api/core.py +295 -287
  3. eodag/api/product/__init__.py +10 -4
  4. eodag/api/product/_assets.py +2 -14
  5. eodag/api/product/_product.py +16 -30
  6. eodag/api/product/drivers/__init__.py +7 -2
  7. eodag/api/product/drivers/base.py +0 -3
  8. eodag/api/product/metadata_mapping.py +12 -31
  9. eodag/api/search_result.py +33 -12
  10. eodag/cli.py +35 -19
  11. eodag/config.py +455 -155
  12. eodag/plugins/apis/base.py +13 -7
  13. eodag/plugins/apis/ecmwf.py +16 -7
  14. eodag/plugins/apis/usgs.py +68 -16
  15. eodag/plugins/authentication/aws_auth.py +25 -7
  16. eodag/plugins/authentication/base.py +10 -1
  17. eodag/plugins/authentication/generic.py +14 -3
  18. eodag/plugins/authentication/header.py +12 -4
  19. eodag/plugins/authentication/keycloak.py +41 -22
  20. eodag/plugins/authentication/oauth.py +11 -1
  21. eodag/plugins/authentication/openid_connect.py +183 -167
  22. eodag/plugins/authentication/qsauth.py +12 -4
  23. eodag/plugins/authentication/sas_auth.py +19 -2
  24. eodag/plugins/authentication/token.py +59 -11
  25. eodag/plugins/authentication/token_exchange.py +19 -19
  26. eodag/plugins/crunch/base.py +7 -2
  27. eodag/plugins/crunch/filter_date.py +8 -11
  28. eodag/plugins/crunch/filter_latest_intersect.py +5 -7
  29. eodag/plugins/crunch/filter_latest_tpl_name.py +2 -5
  30. eodag/plugins/crunch/filter_overlap.py +9 -15
  31. eodag/plugins/crunch/filter_property.py +9 -14
  32. eodag/plugins/download/aws.py +84 -99
  33. eodag/plugins/download/base.py +36 -77
  34. eodag/plugins/download/creodias_s3.py +11 -2
  35. eodag/plugins/download/http.py +134 -109
  36. eodag/plugins/download/s3rest.py +37 -43
  37. eodag/plugins/manager.py +173 -41
  38. eodag/plugins/search/__init__.py +9 -9
  39. eodag/plugins/search/base.py +35 -35
  40. eodag/plugins/search/build_search_result.py +55 -64
  41. eodag/plugins/search/cop_marine.py +113 -32
  42. eodag/plugins/search/creodias_s3.py +20 -8
  43. eodag/plugins/search/csw.py +41 -1
  44. eodag/plugins/search/data_request_search.py +119 -14
  45. eodag/plugins/search/qssearch.py +619 -197
  46. eodag/plugins/search/static_stac_search.py +25 -23
  47. eodag/resources/ext_product_types.json +1 -1
  48. eodag/resources/product_types.yml +211 -56
  49. eodag/resources/providers.yml +1762 -1809
  50. eodag/resources/stac.yml +3 -163
  51. eodag/resources/user_conf_template.yml +134 -119
  52. eodag/rest/config.py +1 -2
  53. eodag/rest/constants.py +0 -1
  54. eodag/rest/core.py +70 -92
  55. eodag/rest/errors.py +181 -0
  56. eodag/rest/server.py +24 -330
  57. eodag/rest/stac.py +105 -630
  58. eodag/rest/types/eodag_search.py +17 -15
  59. eodag/rest/types/queryables.py +5 -14
  60. eodag/rest/types/stac_search.py +18 -13
  61. eodag/rest/utils/rfc3339.py +0 -1
  62. eodag/types/__init__.py +24 -6
  63. eodag/types/download_args.py +14 -5
  64. eodag/types/queryables.py +1 -2
  65. eodag/types/search_args.py +10 -11
  66. eodag/types/whoosh.py +0 -2
  67. eodag/utils/__init__.py +97 -136
  68. eodag/utils/constraints.py +0 -8
  69. eodag/utils/exceptions.py +23 -9
  70. eodag/utils/import_system.py +0 -4
  71. eodag/utils/logging.py +37 -80
  72. eodag/utils/notebook.py +4 -4
  73. eodag/utils/requests.py +13 -23
  74. eodag/utils/rest.py +0 -4
  75. eodag/utils/stac_reader.py +3 -15
  76. {eodag-3.0.0b2.dist-info → eodag-3.0.1.dist-info}/METADATA +41 -24
  77. eodag-3.0.1.dist-info/RECORD +109 -0
  78. {eodag-3.0.0b2.dist-info → eodag-3.0.1.dist-info}/WHEEL +1 -1
  79. {eodag-3.0.0b2.dist-info → eodag-3.0.1.dist-info}/entry_points.txt +1 -0
  80. eodag/resources/constraints/climate-dt.json +0 -13
  81. eodag/resources/constraints/extremes-dt.json +0 -8
  82. eodag-3.0.0b2.dist-info/RECORD +0 -110
  83. {eodag-3.0.0b2.dist-info → eodag-3.0.1.dist-info}/LICENSE +0 -0
  84. {eodag-3.0.0b2.dist-info → eodag-3.0.1.dist-info}/top_level.txt +0 -0
eodag/api/core.py CHANGED
@@ -17,13 +17,25 @@
17
17
  # limitations under the License.
18
18
  from __future__ import annotations
19
19
 
20
+ import datetime
20
21
  import logging
21
22
  import os
22
23
  import re
23
24
  import shutil
24
25
  import tempfile
25
26
  from operator import itemgetter
26
- from typing import TYPE_CHECKING, Any, Dict, Iterator, List, Optional, Set, Tuple, Union
27
+ from typing import (
28
+ TYPE_CHECKING,
29
+ Annotated,
30
+ Any,
31
+ Dict,
32
+ Iterator,
33
+ List,
34
+ Optional,
35
+ Set,
36
+ Tuple,
37
+ Union,
38
+ )
27
39
 
28
40
  import geojson
29
41
  import pkg_resources
@@ -35,11 +47,16 @@ from whoosh.fields import Schema
35
47
  from whoosh.index import create_in, exists_in, open_dir
36
48
  from whoosh.qparser import QueryParser
37
49
 
38
- from eodag.api.product.metadata_mapping import mtd_cfg_as_conversion_and_querypath
50
+ from eodag.api.product.metadata_mapping import (
51
+ ONLINE_STATUS,
52
+ mtd_cfg_as_conversion_and_querypath,
53
+ )
39
54
  from eodag.api.search_result import SearchResult
40
55
  from eodag.config import (
56
+ PLUGINS_TOPICS_KEYS,
41
57
  PluginConfig,
42
58
  SimpleYamlProxyConfig,
59
+ credentials_in_auth,
43
60
  get_ext_product_types_conf,
44
61
  load_default_config,
45
62
  load_stac_provider_config,
@@ -48,10 +65,12 @@ from eodag.config import (
48
65
  override_config_from_file,
49
66
  override_config_from_mapping,
50
67
  provider_config_init,
68
+ share_credentials,
51
69
  )
52
70
  from eodag.plugins.manager import PluginManager
53
71
  from eodag.plugins.search import PreparedSearch
54
72
  from eodag.plugins.search.build_search_result import BuildPostSearchResult
73
+ from eodag.plugins.search.qssearch import PostJsonSearch
55
74
  from eodag.types import model_fields_to_annotated
56
75
  from eodag.types.queryables import CommonQueryables
57
76
  from eodag.types.whoosh import EODAGQueryParser
@@ -66,17 +85,15 @@ from eodag.utils import (
66
85
  MockResponse,
67
86
  _deprecated,
68
87
  copy_deepcopy,
69
- deepcopy,
70
88
  get_geometry_from_various,
71
89
  makedirs,
72
90
  obj_md5sum,
91
+ sort_dict,
73
92
  string_to_jsonpath,
74
93
  uri_to_path,
75
94
  )
76
95
  from eodag.utils.exceptions import (
77
- AuthenticationError,
78
96
  EodagError,
79
- MisconfiguredError,
80
97
  NoMatchingProductType,
81
98
  PluginImplementationError,
82
99
  RequestError,
@@ -96,7 +113,7 @@ if TYPE_CHECKING:
96
113
  from eodag.plugins.search.base import Search
97
114
  from eodag.types import ProviderSortables
98
115
  from eodag.types.download_args import DownloadConf
99
- from eodag.utils import Annotated, DownloadedCallback, ProgressCallback, Unpack
116
+ from eodag.utils import DownloadedCallback, ProgressCallback, Unpack
100
117
 
101
118
  logger = logging.getLogger("eodag.core")
102
119
 
@@ -106,9 +123,7 @@ class EODataAccessGateway:
106
123
  from different types of providers.
107
124
 
108
125
  :param user_conf_file_path: (optional) Path to the user configuration file
109
- :type user_conf_file_path: str
110
126
  :param locations_conf_path: (optional) Path to the locations configuration file
111
- :type locations_conf_path: str
112
127
  """
113
128
 
114
129
  def __init__(
@@ -168,10 +183,15 @@ class EODataAccessGateway:
168
183
  # Second level override: From environment variables
169
184
  override_config_from_env(self.providers_config)
170
185
 
186
+ # share credentials between updated plugins confs
187
+ share_credentials(self.providers_config)
188
+
171
189
  # init updated providers conf
172
- stac_provider_config = load_stac_provider_config()
173
190
  for provider in self.providers_config.keys():
174
- provider_config_init(self.providers_config[provider], stac_provider_config)
191
+ provider_config_init(
192
+ self.providers_config[provider],
193
+ load_stac_provider_config(),
194
+ )
175
195
 
176
196
  # re-build _plugins_manager using up-to-date providers_config
177
197
  self._plugins_manager.rebuild(self.providers_config)
@@ -217,7 +237,6 @@ class EODataAccessGateway:
217
237
  os.path.join(self.conf_dir, "shp"),
218
238
  )
219
239
  self.set_locations_conf(locations_conf_path)
220
- self.search_errors: Set = set()
221
240
 
222
241
  def get_version(self) -> str:
223
242
  """Get eodag package version"""
@@ -312,7 +331,6 @@ class EODataAccessGateway:
312
331
 
313
332
  :param provider: The name of the provider that should be considered as the
314
333
  preferred provider to be used for this instance
315
- :type provider: str
316
334
  """
317
335
  if provider not in self.available_providers():
318
336
  raise UnsupportedProvider(
@@ -328,7 +346,6 @@ class EODataAccessGateway:
328
346
  products, along with its priority.
329
347
 
330
348
  :returns: The provider with the maximum priority and its priority
331
- :rtype: tuple(str, int)
332
349
  """
333
350
  providers_with_priority = [
334
351
  (provider, conf.priority)
@@ -337,15 +354,24 @@ class EODataAccessGateway:
337
354
  preferred, priority = max(providers_with_priority, key=itemgetter(1))
338
355
  return preferred, priority
339
356
 
340
- def update_providers_config(self, yaml_conf: str) -> None:
357
+ def update_providers_config(
358
+ self,
359
+ yaml_conf: Optional[str] = None,
360
+ dict_conf: Optional[Dict[str, Any]] = None,
361
+ ) -> None:
341
362
  """Update providers configuration with given input.
342
363
  Can be used to add a provider to existing configuration or update
343
364
  an existing one.
344
365
 
345
366
  :param yaml_conf: YAML formated provider configuration
346
- :type yaml_conf: str
367
+ :param dict_conf: provider configuration as dictionary in place of ``yaml_conf``
347
368
  """
348
- conf_update = yaml.safe_load(yaml_conf)
369
+ if dict_conf is not None:
370
+ conf_update = dict_conf
371
+ elif yaml_conf is not None:
372
+ conf_update = yaml.safe_load(yaml_conf)
373
+ else:
374
+ return None
349
375
 
350
376
  # restore the pruned configuration
351
377
  for provider in list(self._pruned_providers_config.keys()):
@@ -358,48 +384,86 @@ class EODataAccessGateway:
358
384
  provider
359
385
  )
360
386
 
361
- # check if metada-mapping as already been built as jsonpath in providers_config
362
- for provider, provider_conf in conf_update.items():
363
- if (
364
- provider in self.providers_config
365
- and "metadata_mapping" in provider_conf.get("search", {})
366
- ):
367
- search_plugin_key = "search"
368
- elif (
369
- provider in self.providers_config
370
- and "metadata_mapping" in provider_conf.get("api", {})
371
- ):
372
- search_plugin_key = "api"
373
- else:
374
- continue
375
- # get some already configured value
376
- configured_metadata_mapping = getattr(
377
- self.providers_config[provider], search_plugin_key
378
- ).metadata_mapping
379
- some_configured_value = next(iter(configured_metadata_mapping.values()))
380
- # check if the configured value has already been built as jsonpath
381
- if (
382
- isinstance(some_configured_value, list)
383
- and isinstance(some_configured_value[1], tuple)
384
- or isinstance(some_configured_value, tuple)
385
- ):
386
- # also build as jsonpath the incoming conf
387
- mtd_cfg_as_conversion_and_querypath(
388
- deepcopy(
389
- conf_update[provider][search_plugin_key]["metadata_mapping"]
390
- ),
391
- conf_update[provider][search_plugin_key]["metadata_mapping"],
392
- )
393
-
394
387
  override_config_from_mapping(self.providers_config, conf_update)
395
388
 
396
- stac_provider_config = load_stac_provider_config()
389
+ # share credentials between updated plugins confs
390
+ share_credentials(self.providers_config)
391
+
397
392
  for provider in conf_update.keys():
398
- provider_config_init(self.providers_config[provider], stac_provider_config)
393
+ provider_config_init(
394
+ self.providers_config[provider],
395
+ load_stac_provider_config(),
396
+ )
399
397
  setattr(self.providers_config[provider], "product_types_fetched", False)
400
398
  # re-create _plugins_manager using up-to-date providers_config
401
399
  self._plugins_manager.build_product_type_to_provider_config_map()
402
400
 
401
+ def add_provider(
402
+ self,
403
+ name: str,
404
+ url: Optional[str] = None,
405
+ priority: Optional[int] = None,
406
+ search: Dict[str, Any] = {"type": "StacSearch"},
407
+ products: Dict[str, Any] = {
408
+ GENERIC_PRODUCT_TYPE: {"productType": "{productType}"}
409
+ },
410
+ download: Dict[str, Any] = {"type": "HTTPDownload", "auth_error_code": 401},
411
+ **kwargs: Dict[str, Any],
412
+ ):
413
+ """Adds a new provider.
414
+
415
+ ``search``, ``products`` & ``download`` already have default values that will be
416
+ updated (not replaced), with user provided ones:
417
+
418
+ * ``search`` : ``{"type": "StacSearch"}``
419
+ * ``products`` : ``{"GENERIC_PRODUCT_TYPE": {"productType": "{productType}"}}``
420
+ * ``download`` : ``{"type": "HTTPDownload", "auth_error_code": 401}``
421
+
422
+ :param name: Name of provider
423
+ :param url: Provider url, also used as ``search["api_endpoint"]`` if not defined
424
+ :param priority: Provider priority. If None, provider will be set as preferred (highest priority)
425
+ :param search: Search :class:`~eodag.config.PluginConfig` mapping
426
+ :param products: Provider product types mapping
427
+ :param download: Download :class:`~eodag.config.PluginConfig` mapping
428
+ :param kwargs: Additional :class:`~eodag.config.ProviderConfig` mapping
429
+ """
430
+ conf_dict: Dict[str, Any] = {
431
+ name: {
432
+ "url": url,
433
+ "search": {"type": "StacSearch", **search},
434
+ "products": {
435
+ GENERIC_PRODUCT_TYPE: {"productType": "{productType}"},
436
+ **products,
437
+ },
438
+ "download": {
439
+ "type": "HTTPDownload",
440
+ "auth_error_code": 401,
441
+ **download,
442
+ },
443
+ **kwargs,
444
+ }
445
+ }
446
+ if priority is not None:
447
+ conf_dict[name]["priority"] = priority
448
+ # if provided, use url as default search api_endpoint
449
+ if (
450
+ url
451
+ and conf_dict[name].get("search", {})
452
+ and not conf_dict[name]["search"].get("api_endpoint")
453
+ ):
454
+ conf_dict[name]["search"]["api_endpoint"] = url
455
+
456
+ # api plugin usage: remove unneeded search/download/auth plugin conf
457
+ if conf_dict[name].get("api"):
458
+ for k in PLUGINS_TOPICS_KEYS:
459
+ if k != "api":
460
+ conf_dict[name].pop(k, None)
461
+
462
+ self.update_providers_config(dict_conf=conf_dict)
463
+
464
+ if priority is None:
465
+ self.set_preferred_provider(name)
466
+
403
467
  def _prune_providers_list(self) -> None:
404
468
  """Removes from config providers needing auth that have no credentials set."""
405
469
  update_needed = False
@@ -421,12 +485,7 @@ class EODataAccessGateway:
421
485
 
422
486
  # check authentication
423
487
  if hasattr(conf, "api") and getattr(conf.api, "need_auth", False):
424
- credentials_exist = any(
425
- [
426
- cred is not None
427
- for cred in getattr(conf.api, "credentials", {}).values()
428
- ]
429
- )
488
+ credentials_exist = credentials_in_auth(conf.api)
430
489
  if not credentials_exist:
431
490
  # credentials needed but not found
432
491
  self._pruned_providers_config[provider] = self.providers_config.pop(
@@ -438,7 +497,7 @@ class EODataAccessGateway:
438
497
  provider,
439
498
  )
440
499
  elif hasattr(conf, "search") and getattr(conf.search, "need_auth", False):
441
- if not hasattr(conf, "auth"):
500
+ if not hasattr(conf, "auth") and not hasattr(conf, "search_auth"):
442
501
  # credentials needed but no auth plugin was found
443
502
  self._pruned_providers_config[provider] = self.providers_config.pop(
444
503
  provider
@@ -449,11 +508,13 @@ class EODataAccessGateway:
449
508
  provider,
450
509
  )
451
510
  continue
452
- credentials_exist = any(
453
- [
454
- cred is not None
455
- for cred in getattr(conf.auth, "credentials", {}).values()
456
- ]
511
+ credentials_exist = (
512
+ hasattr(conf, "search_auth")
513
+ and credentials_in_auth(conf.search_auth)
514
+ ) or (
515
+ not hasattr(conf, "search_auth")
516
+ and hasattr(conf, "auth")
517
+ and credentials_in_auth(conf.auth)
457
518
  )
458
519
  if not credentials_exist:
459
520
  # credentials needed but not found
@@ -502,7 +563,6 @@ class EODataAccessGateway:
502
563
  attr: FRA
503
564
 
504
565
  :param locations_conf_path: Path to the locations configuration file
505
- :type locations_conf_path: str
506
566
  """
507
567
  if os.path.isfile(locations_conf_path):
508
568
  locations_config = load_yml_config(locations_conf_path)
@@ -525,12 +585,9 @@ class EODataAccessGateway:
525
585
 
526
586
  :param provider: (optional) The name of a provider that must support the product
527
587
  types we are about to list
528
- :type provider: str
529
588
  :param fetch_providers: (optional) Whether to fetch providers for new product
530
589
  types or not
531
- :type fetch_providers: bool
532
590
  :returns: The list of the product types that can be accessed using eodag.
533
- :rtype: list(dict)
534
591
  :raises: :class:`~eodag.utils.exceptions.UnsupportedProvider`
535
592
  """
536
593
  if fetch_providers:
@@ -572,22 +629,32 @@ class EODataAccessGateway:
572
629
  def fetch_product_types_list(self, provider: Optional[str] = None) -> None:
573
630
  """Fetch product types list and update if needed
574
631
 
575
- :param provider: (optional) The name of a provider for which product types list
576
- should be updated. Defaults to all providers (None value).
577
- :type provider: str
632
+ :param provider: The name of a provider or provider-group for which product types
633
+ list should be updated. Defaults to all providers (None value).
578
634
  """
635
+ providers_to_fetch = list(self.providers_config.keys())
636
+ # check if some providers are grouped under a group name which is not a provider name
579
637
  if provider is not None and provider not in self.providers_config:
580
- return
638
+ providers_to_fetch = [
639
+ p
640
+ for p, pconf in self.providers_config.items()
641
+ if provider == getattr(pconf, "group", None)
642
+ ]
643
+ if providers_to_fetch:
644
+ logger.info(
645
+ f"Fetch product types for {provider} group: {', '.join(providers_to_fetch)}"
646
+ )
647
+ else:
648
+ return None
649
+ elif provider is not None:
650
+ providers_to_fetch = [provider]
581
651
 
582
652
  # providers discovery confs that are fetchable
583
653
  providers_discovery_configs_fetchable: Dict[str, Any] = {}
584
654
  # check if any provider has not already been fetched for product types
585
655
  already_fetched = True
586
- for provider_to_fetch, provider_config in (
587
- {provider: self.providers_config[provider]}.items()
588
- if provider
589
- else self.providers_config.items()
590
- ):
656
+ for provider_to_fetch in providers_to_fetch:
657
+ provider_config = self.providers_config[provider_to_fetch]
591
658
  # get discovery conf
592
659
  if hasattr(provider_config, "search"):
593
660
  provider_search_config = provider_config.search
@@ -709,13 +776,20 @@ class EODataAccessGateway:
709
776
  ) -> Optional[Dict[str, Any]]:
710
777
  """Fetch providers for product types
711
778
 
712
- :param provider: (optional) The name of a provider to fetch. Defaults to all
713
- providers (None value).
714
- :type provider: str
779
+ :param provider: The name of a provider or provider-group to fetch. Defaults to
780
+ all providers (None value).
715
781
  :returns: external product types configuration
716
- :rtype: dict
717
782
  """
718
- if provider and provider not in self.providers_config:
783
+ grouped_providers = [
784
+ p
785
+ for p, provider_config in self.providers_config.items()
786
+ if provider == getattr(provider_config, "group", None)
787
+ ]
788
+ if provider and provider not in self.providers_config and grouped_providers:
789
+ logger.info(
790
+ f"Discover product types for {provider} group: {', '.join(grouped_providers)}"
791
+ )
792
+ elif provider and provider not in self.providers_config:
719
793
  raise UnsupportedProvider(
720
794
  f"The requested provider is not (yet) supported: {provider}"
721
795
  )
@@ -724,7 +798,9 @@ class EODataAccessGateway:
724
798
  p
725
799
  for p in (
726
800
  [
727
- provider,
801
+ p
802
+ for p in self.providers_config
803
+ if p in grouped_providers + [provider]
728
804
  ]
729
805
  if provider
730
806
  else self.available_providers()
@@ -738,29 +814,28 @@ class EODataAccessGateway:
738
814
  search_plugin_config = self.providers_config[provider].api
739
815
  else:
740
816
  return None
741
- if getattr(search_plugin_config, "discover_product_types", None):
817
+ if getattr(search_plugin_config, "discover_product_types", {}).get(
818
+ "fetch_url", None
819
+ ):
742
820
  search_plugin: Union[Search, Api] = next(
743
821
  self._plugins_manager.get_search_plugins(provider=provider)
744
822
  )
823
+ # check after plugin init if still fetchable
824
+ if not getattr(search_plugin.config, "discover_product_types", {}).get(
825
+ "fetch_url"
826
+ ):
827
+ continue
745
828
  # append auth to search plugin if needed
746
829
  if getattr(search_plugin.config, "need_auth", False):
747
- auth_plugin = self._plugins_manager.get_auth_plugin(
748
- search_plugin.provider
749
- )
750
- if auth_plugin and callable(
751
- getattr(auth_plugin, "authenticate", None)
830
+ if auth := self._plugins_manager.get_auth(
831
+ search_plugin.provider,
832
+ getattr(search_plugin.config, "api_endpoint", None),
833
+ search_plugin.config,
752
834
  ):
753
- try:
754
- kwargs["auth"] = auth_plugin.authenticate()
755
- except (AuthenticationError, MisconfiguredError) as e:
756
- logger.warning(
757
- f"Could not authenticate on {provider}: {str(e)}"
758
- )
759
- ext_product_types_conf[provider] = None
760
- continue
835
+ kwargs["auth"] = auth
761
836
  else:
762
- logger.warning(
763
- f"Could not authenticate on {provider} using {auth_plugin} plugin"
837
+ logger.debug(
838
+ f"Could not authenticate on {provider} for product types discovery"
764
839
  )
765
840
  ext_product_types_conf[provider] = None
766
841
  continue
@@ -769,7 +844,7 @@ class EODataAccessGateway:
769
844
  **kwargs
770
845
  )
771
846
 
772
- return ext_product_types_conf
847
+ return sort_dict(ext_product_types_conf)
773
848
 
774
849
  def update_product_types_list(
775
850
  self, ext_product_types_conf: Dict[str, Optional[Dict[str, Dict[str, Any]]]]
@@ -777,7 +852,6 @@ class EODataAccessGateway:
777
852
  """Update eodag product types list
778
853
 
779
854
  :param ext_product_types_conf: external product types configuration
780
- :type ext_product_types_conf: dict
781
855
  """
782
856
  for provider, new_product_types_conf in ext_product_types_conf.items():
783
857
  if new_product_types_conf and provider in self.providers_config:
@@ -787,7 +861,9 @@ class EODataAccessGateway:
787
861
  ) or getattr(self.providers_config[provider], "api", None)
788
862
  if search_plugin_config is None:
789
863
  continue
790
- if not hasattr(search_plugin_config, "discover_product_types"):
864
+ if not getattr(
865
+ search_plugin_config, "discover_product_types", {}
866
+ ).get("fetch_url", None):
791
867
  # conf has been updated and provider product types are no more discoverable
792
868
  continue
793
869
  provider_products_config = (
@@ -846,7 +922,7 @@ class EODataAccessGateway:
846
922
  new_product_types.append(new_product_type)
847
923
  if new_product_types:
848
924
  logger.debug(
849
- f"Added product types {str(new_product_types)} for {provider}"
925
+ f"Added {len(new_product_types)} product types for {provider}"
850
926
  )
851
927
 
852
928
  elif provider not in self.providers_config:
@@ -870,12 +946,9 @@ class EODataAccessGateway:
870
946
  priority level.
871
947
 
872
948
  :param product_type: (optional) Only list providers configured for this product_type
873
- :type product_type: Optional[str]
874
949
  :param by_group: (optional) If set to True, list groups when available instead
875
950
  of providers, mixed with other providers
876
- :type by_group: bool
877
951
  :returns: the sorted list of the available providers or groups
878
- :rtype: List[str]
879
952
  """
880
953
 
881
954
  if product_type:
@@ -909,9 +982,7 @@ class EODataAccessGateway:
909
982
 
910
983
  :param alias_or_id: Alias of the product type. If an existing ID is given, this
911
984
  method will directly return the given value.
912
- :type alias_or_id: str
913
985
  :returns: Internal name of the product type.
914
- :rtype: str
915
986
  """
916
987
  product_types = [
917
988
  k
@@ -939,9 +1010,7 @@ class EODataAccessGateway:
939
1010
  given product type, its ID is returned instead.
940
1011
 
941
1012
  :param product_type: product type ID
942
- :type product_type: str
943
1013
  :returns: Alias of the product type or its ID if no alias has been defined for it.
944
- :rtype: str
945
1014
  """
946
1015
  if product_type not in self.product_types_config:
947
1016
  raise NoMatchingProductType(product_type)
@@ -972,31 +1041,18 @@ class EODataAccessGateway:
972
1041
 
973
1042
  :param free_text: Whoosh-compatible free text search filter used to search
974
1043
  accross all the following parameters
975
- :type free_text: Optional[str]
976
1044
  :param intersect: Join results for each parameter using INTERSECT instead of UNION.
977
- :type intersect: bool
978
1045
  :param instrument: Instrument parameter.
979
- :type instrument: Optional[str]
980
1046
  :param platform: Platform parameter.
981
- :type platform: Optional[str]
982
1047
  :param platformSerialIdentifier: Platform serial identifier parameter.
983
- :type platformSerialIdentifier: Optional[str]
984
1048
  :param processingLevel: Processing level parameter.
985
- :type processingLevel: Optional[str]
986
1049
  :param sensorType: Sensor type parameter.
987
- :type sensorType: Optional[str]
988
1050
  :param keywords: Keywords parameter.
989
- :type keywords: Optional[str]
990
1051
  :param abstract: Abstract parameter.
991
- :type abstract: Optional[str]
992
1052
  :param title: Title parameter.
993
- :type title: Optional[str]
994
1053
  :param missionStartDate: start date for datetime filtering. Not used by free_text
995
- :type missionStartDate: Optional[str]
996
1054
  :param missionEndDate: end date for datetime filtering. Not used by free_text
997
- :type missionEndDate: Optional[str]
998
1055
  :returns: The best match for the given parameters.
999
- :rtype: List[str]
1000
1056
  :raises: :class:`~eodag.utils.exceptions.NoMatchingProductType`
1001
1057
  """
1002
1058
  if productType := kwargs.get("productType"):
@@ -1038,20 +1094,28 @@ class EODataAccessGateway:
1038
1094
 
1039
1095
  # datetime filtering
1040
1096
  if missionStartDate or missionEndDate:
1097
+ min_aware = datetime.datetime.min.replace(tzinfo=datetime.timezone.utc)
1098
+ max_aware = datetime.datetime.max.replace(tzinfo=datetime.timezone.utc)
1041
1099
  guesses = [
1042
1100
  g
1043
1101
  for g in guesses
1044
1102
  if (
1045
- not missionEndDate
1046
- or g.get("missionStartDate")
1047
- and rfc3339_str_to_datetime(g["missionStartDate"])
1048
- <= rfc3339_str_to_datetime(missionEndDate)
1049
- )
1050
- and (
1051
- not missionStartDate
1052
- or g.get("missionEndDate")
1053
- and rfc3339_str_to_datetime(g["missionEndDate"])
1054
- >= rfc3339_str_to_datetime(missionStartDate)
1103
+ max(
1104
+ rfc3339_str_to_datetime(missionStartDate)
1105
+ if missionStartDate
1106
+ else min_aware,
1107
+ rfc3339_str_to_datetime(g["missionStartDate"])
1108
+ if g.get("missionStartDate")
1109
+ else min_aware,
1110
+ )
1111
+ <= min(
1112
+ rfc3339_str_to_datetime(missionEndDate)
1113
+ if missionEndDate
1114
+ else max_aware,
1115
+ rfc3339_str_to_datetime(g["missionEndDate"])
1116
+ if g.get("missionEndDate")
1117
+ else max_aware,
1118
+ )
1055
1119
  )
1056
1120
  ]
1057
1121
 
@@ -1083,21 +1147,16 @@ class EODataAccessGateway:
1083
1147
  Only if the request fails for all available providers, an error will be thrown.
1084
1148
 
1085
1149
  :param page: (optional) The page number to return
1086
- :type page: int
1087
1150
  :param items_per_page: (optional) The number of results that must appear in one single
1088
1151
  page
1089
- :type items_per_page: int
1090
1152
  :param raise_errors: (optional) When an error occurs when searching, if this is set to
1091
1153
  True, the error is raised
1092
- :type raise_errors: bool
1093
1154
  :param start: (optional) Start sensing time in ISO 8601 format (e.g. "1990-11-26",
1094
1155
  "1990-11-26T14:30:10.153Z", "1990-11-26T14:30:10+02:00", ...).
1095
1156
  If no time offset is given, the time is assumed to be given in UTC.
1096
- :type start: str
1097
1157
  :param end: (optional) End sensing time in ISO 8601 format (e.g. "1990-11-26",
1098
1158
  "1990-11-26T14:30:10.153Z", "1990-11-26T14:30:10+02:00", ...).
1099
1159
  If no time offset is given, the time is assumed to be given in UTC.
1100
- :type end: str
1101
1160
  :param geom: (optional) Search area that can be defined in different ways:
1102
1161
 
1103
1162
  * with a Shapely geometry object:
@@ -1107,24 +1166,22 @@ class EODataAccessGateway:
1107
1166
  * with a bounding box as list of float:
1108
1167
  ``[lonmin, latmin, lonmax, latmax]``
1109
1168
  * with a WKT str
1110
- :type geom: Union[str, dict, shapely.geometry.base.BaseGeometry]
1111
1169
  :param locations: (optional) Location filtering by name using locations configuration
1112
1170
  ``{"<location_name>"="<attr_regex>"}``. For example, ``{"country"="PA."}`` will use
1113
1171
  the geometry of the features having the property ISO3 starting with
1114
1172
  'PA' such as Panama and Pakistan in the shapefile configured with
1115
1173
  name=country and attr=ISO3
1116
- :type locations: dict
1117
1174
  :param provider: (optional) the provider to be used. If set, search fallback will be disabled.
1118
1175
  If not set, the configured preferred provider will be used at first
1119
1176
  before trying others until finding results.
1120
- :type provider: str
1121
1177
  :param count: (optional) Whether to run a query with a count request or not
1122
- :type count: bool
1123
1178
  :param kwargs: Some other criteria that will be used to do the search,
1124
1179
  using paramaters compatibles with the provider
1125
- :type kwargs: Union[int, str, bool, dict]
1126
1180
  :returns: A collection of EO products matching the criteria
1127
- :rtype: :class:`~eodag.api.search_result.SearchResult`
1181
+
1182
+ .. versionchanged:: v3.0.0b1
1183
+ ``search()`` method now returns only a single :class:`~eodag.api.search_result.SearchResult`
1184
+ instead of a 2 values tuple.
1128
1185
 
1129
1186
  .. note::
1130
1187
  The search interfaces, which are implemented as plugins, are required to
@@ -1154,7 +1211,7 @@ class EODataAccessGateway:
1154
1211
  items_per_page=items_per_page,
1155
1212
  )
1156
1213
 
1157
- self.search_errors = set()
1214
+ errors: List[Tuple[str, Exception]] = []
1158
1215
  # Loop over available providers and return the first non-empty results
1159
1216
  for i, search_plugin in enumerate(search_plugins):
1160
1217
  search_plugin.clear()
@@ -1164,17 +1221,19 @@ class EODataAccessGateway:
1164
1221
  raise_errors=raise_errors,
1165
1222
  **search_kwargs,
1166
1223
  )
1224
+ errors.extend(search_results.errors)
1167
1225
  if len(search_results) == 0 and i < len(search_plugins) - 1:
1168
1226
  logger.warning(
1169
1227
  f"No result could be obtained from provider {search_plugin.provider}, "
1170
1228
  "we will try to get the data from another provider",
1171
1229
  )
1172
1230
  elif len(search_results) > 0:
1231
+ search_results.errors = errors
1173
1232
  return search_results
1174
1233
 
1175
1234
  if i > 1:
1176
1235
  logger.error("No result could be obtained from any available provider")
1177
- return SearchResult([], 0) if count else SearchResult([])
1236
+ return SearchResult([], 0, errors) if count else SearchResult([], errors=errors)
1178
1237
 
1179
1238
  def search_iter_page(
1180
1239
  self,
@@ -1188,15 +1247,12 @@ class EODataAccessGateway:
1188
1247
  """Iterate over the pages of a products search.
1189
1248
 
1190
1249
  :param items_per_page: (optional) The number of results requested per page
1191
- :type items_per_page: int
1192
1250
  :param start: (optional) Start sensing time in ISO 8601 format (e.g. "1990-11-26",
1193
1251
  "1990-11-26T14:30:10.153Z", "1990-11-26T14:30:10+02:00", ...).
1194
1252
  If no time offset is given, the time is assumed to be given in UTC.
1195
- :type start: str
1196
1253
  :param end: (optional) End sensing time in ISO 8601 format (e.g. "1990-11-26",
1197
1254
  "1990-11-26T14:30:10.153Z", "1990-11-26T14:30:10+02:00", ...).
1198
1255
  If no time offset is given, the time is assumed to be given in UTC.
1199
- :type end: str
1200
1256
  :param geom: (optional) Search area that can be defined in different ways:
1201
1257
 
1202
1258
  * with a Shapely geometry object:
@@ -1206,19 +1262,15 @@ class EODataAccessGateway:
1206
1262
  * with a bounding box as list of float:
1207
1263
  ``[lonmin, latmin, lonmax, latmax]``
1208
1264
  * with a WKT str
1209
- :type geom: Union[str, dict, shapely.geometry.base.BaseGeometry]
1210
1265
  :param locations: (optional) Location filtering by name using locations configuration
1211
1266
  ``{"<location_name>"="<attr_regex>"}``. For example, ``{"country"="PA."}`` will use
1212
1267
  the geometry of the features having the property ISO3 starting with
1213
1268
  'PA' such as Panama and Pakistan in the shapefile configured with
1214
1269
  name=country and attr=ISO3
1215
- :type locations: dict
1216
1270
  :param kwargs: Some other criteria that will be used to do the search,
1217
1271
  using paramaters compatibles with the provider
1218
- :type kwargs: Union[int, str, bool, dict]
1219
1272
  :returns: An iterator that yields page per page a collection of EO products
1220
1273
  matching the criteria
1221
- :rtype: Iterator[:class:`~eodag.api.search_result.SearchResult`]
1222
1274
  """
1223
1275
  search_plugins, search_kwargs = self._prepare_search(
1224
1276
  start=start, end=end, geom=geom, locations=locations, **kwargs
@@ -1253,15 +1305,11 @@ class EODataAccessGateway:
1253
1305
  """Iterate over the pages of a products search using a given search plugin.
1254
1306
 
1255
1307
  :param items_per_page: (optional) The number of results requested per page
1256
- :type items_per_page: int
1257
1308
  :param kwargs: Some other criteria that will be used to do the search,
1258
1309
  using parameters compatibles with the provider
1259
- :type kwargs: Union[int, str, bool, dict]
1260
1310
  :param search_plugin: search plugin to be used
1261
- :type search_plugin: eodag.plugins.search.base.Search
1262
1311
  :returns: An iterator that yields page per page a collection of EO products
1263
1312
  matching the criteria
1264
- :rtype: Iterator[:class:`~eodag.api.search_result.SearchResult`]
1265
1313
  """
1266
1314
 
1267
1315
  iteration = 1
@@ -1390,15 +1438,12 @@ class EODataAccessGateway:
1390
1438
  matching the search criteria. If this number is not
1391
1439
  available, a default value of 50 is used instead.
1392
1440
  items_per_page can also be set to any arbitrary value.
1393
- :type items_per_page: int
1394
1441
  :param start: (optional) Start sensing time in ISO 8601 format (e.g. "1990-11-26",
1395
1442
  "1990-11-26T14:30:10.153Z", "1990-11-26T14:30:10+02:00", ...).
1396
1443
  If no time offset is given, the time is assumed to be given in UTC.
1397
- :type start: str
1398
1444
  :param end: (optional) End sensing time in ISO 8601 format (e.g. "1990-11-26",
1399
1445
  "1990-11-26T14:30:10.153Z", "1990-11-26T14:30:10+02:00", ...).
1400
1446
  If no time offset is given, the time is assumed to be given in UTC.
1401
- :type end: str
1402
1447
  :param geom: (optional) Search area that can be defined in different ways:
1403
1448
 
1404
1449
  * with a Shapely geometry object:
@@ -1408,19 +1453,15 @@ class EODataAccessGateway:
1408
1453
  * with a bounding box as list of float:
1409
1454
  ``[lonmin, latmin, lonmax, latmax]``
1410
1455
  * with a WKT str
1411
- :type geom: Union[str, dict, shapely.geometry.base.BaseGeometry]
1412
1456
  :param locations: (optional) Location filtering by name using locations configuration
1413
1457
  ``{"<location_name>"="<attr_regex>"}``. For example, ``{"country"="PA."}`` will use
1414
1458
  the geometry of the features having the property ISO3 starting with
1415
1459
  'PA' such as Panama and Pakistan in the shapefile configured with
1416
1460
  name=country and attr=ISO3
1417
- :type locations: dict
1418
1461
  :param kwargs: Some other criteria that will be used to do the search,
1419
1462
  using parameters compatible with the provider
1420
- :type kwargs: Union[int, str, bool, dict]
1421
1463
  :returns: An iterator that yields page per page a collection of EO products
1422
1464
  matching the criteria
1423
- :rtype: Iterator[:class:`~eodag.api.search_result.SearchResult`]
1424
1465
  """
1425
1466
  # Get the search plugin and the maximized value
1426
1467
  # of items_per_page if defined for the provider used.
@@ -1452,7 +1493,7 @@ class EODataAccessGateway:
1452
1493
  )
1453
1494
  or DEFAULT_MAX_ITEMS_PER_PAGE
1454
1495
  )
1455
- logger.debug(
1496
+ logger.info(
1456
1497
  "Searching for all the products with provider %s and a maximum of %s "
1457
1498
  "items per page.",
1458
1499
  search_plugin.provider,
@@ -1509,22 +1550,18 @@ class EODataAccessGateway:
1509
1550
  perform the search, if this information is available
1510
1551
 
1511
1552
  :param uid: The uid of the EO product
1512
- :type uid: str
1513
1553
  :param provider: (optional) The provider on which to search the product.
1514
1554
  This may be useful for performance reasons when the user
1515
1555
  knows this product is available on the given provider
1516
- :type provider: str
1517
1556
  :param kwargs: Search criteria to help finding the right product
1518
- :type kwargs: Any
1519
1557
  :returns: A search result with one EO product or None at all
1520
- :rtype: :class:`~eodag.api.search_result.SearchResult`
1521
1558
  """
1522
1559
  product_type = kwargs.get("productType", None)
1523
1560
  if product_type is not None:
1524
1561
  try:
1525
1562
  product_type = self.get_product_type_from_alias(product_type)
1526
1563
  except NoMatchingProductType:
1527
- logger.warning("product type %s not found", product_type)
1564
+ logger.debug("product type %s not found", product_type)
1528
1565
  get_search_plugins_kwargs = dict(provider=provider, product_type=product_type)
1529
1566
  search_plugins = self._plugins_manager.get_search_plugins(
1530
1567
  **get_search_plugins_kwargs
@@ -1547,7 +1584,7 @@ class EODataAccessGateway:
1547
1584
  "max_items_per_page", DEFAULT_MAX_ITEMS_PER_PAGE
1548
1585
  )
1549
1586
  kwargs.update(items_per_page=items_per_page)
1550
- if isinstance(plugin, BuildPostSearchResult):
1587
+ if isinstance(plugin, PostJsonSearch):
1551
1588
  kwargs.update(
1552
1589
  items_per_page=items_per_page,
1553
1590
  _dc_qs=_dc_qs,
@@ -1565,9 +1602,10 @@ class EODataAccessGateway:
1565
1602
  **kwargs,
1566
1603
  ):
1567
1604
  results.data.extend(page_results.data)
1568
- except Exception:
1605
+ except Exception as e:
1569
1606
  if kwargs.get("raise_errors"):
1570
1607
  raise
1608
+ logger.warning(e)
1571
1609
  continue
1572
1610
 
1573
1611
  # try using crunch to get unique result
@@ -1597,20 +1635,20 @@ class EODataAccessGateway:
1597
1635
  plugins = self._plugins_manager.get_search_plugins(provider=provider)
1598
1636
  plugin = next(plugins)
1599
1637
 
1638
+ # check after plugin init if still fetchable
1639
+ if not getattr(plugin.config, "discover_product_types", {}).get("fetch_url"):
1640
+ return None
1641
+
1600
1642
  kwargs: Dict[str, Any] = {"productType": product_type}
1601
1643
 
1602
1644
  # append auth if needed
1603
1645
  if getattr(plugin.config, "need_auth", False):
1604
- auth_plugin = self._plugins_manager.get_auth_plugin(plugin.provider)
1605
- if auth_plugin and callable(getattr(auth_plugin, "authenticate", None)):
1606
- try:
1607
- kwargs["auth"] = auth_plugin.authenticate()
1608
- except (AuthenticationError, MisconfiguredError) as e:
1609
- logger.warning(f"Could not authenticate on {provider}: {str(e)}")
1610
- else:
1611
- logger.warning(
1612
- f"Could not authenticate on {provider} using {auth_plugin} plugin"
1613
- )
1646
+ if auth := self._plugins_manager.get_auth(
1647
+ plugin.provider,
1648
+ getattr(plugin.config, "api_endpoint", None),
1649
+ plugin.config,
1650
+ ):
1651
+ kwargs["auth"] = auth
1614
1652
 
1615
1653
  product_type_config = plugin.discover_product_types(**kwargs)
1616
1654
  self.update_product_types_list({provider: product_type_config})
@@ -1641,25 +1679,18 @@ class EODataAccessGateway:
1641
1679
  :param start: (optional) Start sensing time in ISO 8601 format (e.g. "1990-11-26",
1642
1680
  "1990-11-26T14:30:10.153Z", "1990-11-26T14:30:10+02:00", ...).
1643
1681
  If no time offset is given, the time is assumed to be given in UTC.
1644
- :type start: str
1645
1682
  :param end: (optional) End sensing time in ISO 8601 format (e.g. "1990-11-26",
1646
1683
  "1990-11-26T14:30:10.153Z", "1990-11-26T14:30:10+02:00", ...).
1647
1684
  If no time offset is given, the time is assumed to be given in UTC.
1648
- :type end: str
1649
1685
  :param geom: (optional) Search area that can be defined in different ways (see search)
1650
- :type geom: Union[str, dict, shapely.geometry.base.BaseGeometry]
1651
1686
  :param locations: (optional) Location filtering by name using locations configuration
1652
- :type locations: dict
1653
1687
  :param provider: provider to be used, if no provider is given or the product type
1654
1688
  is not available for the provider, the preferred provider is used
1655
- :type provider: str
1656
1689
  :param kwargs: Some other criteria
1657
1690
  * id and/or a provider for a search by
1658
1691
  * search criteria to guess the product type
1659
1692
  * other criteria compatible with the provider
1660
- :type kwargs: Any
1661
1693
  :returns: Search plugins list and the prepared kwargs to make a query.
1662
- :rtype: tuple(list, dict)
1663
1694
  """
1664
1695
  product_type = kwargs.get("productType", None)
1665
1696
  if product_type is None:
@@ -1725,18 +1756,15 @@ class EODataAccessGateway:
1725
1756
  product_type
1726
1757
  not in self._plugins_manager.product_type_to_provider_config_map.keys()
1727
1758
  ):
1728
- logger.debug(
1729
- f"Fetching external product types sources to find {product_type} product type"
1730
- )
1731
1759
  if provider:
1732
1760
  # Try to get specific product type from external provider
1761
+ logger.debug(f"Fetching {provider} to find {product_type} product type")
1733
1762
  self._fetch_external_product_type(provider, product_type)
1734
- if (
1735
- not provider
1736
- or product_type
1737
- not in self._plugins_manager.product_type_to_provider_config_map.keys()
1738
- ):
1763
+ if not provider:
1739
1764
  # no provider or still not found -> fetch all external product types
1765
+ logger.debug(
1766
+ f"Fetching external product types sources to find {product_type} product type"
1767
+ )
1740
1768
  self.fetch_product_types_list()
1741
1769
 
1742
1770
  preferred_provider = self.get_preferred_provider()[0]
@@ -1759,12 +1787,10 @@ class EODataAccessGateway:
1759
1787
  provider = preferred_provider
1760
1788
  providers = [plugin.provider for plugin in search_plugins]
1761
1789
  if provider not in providers:
1762
- logger.warning(
1763
- "Product type '%s' is not available with provider '%s'. "
1764
- "Searching it on provider '%s' instead.",
1790
+ logger.debug(
1791
+ "Product type '%s' is not available with preferred provider '%s'.",
1765
1792
  product_type,
1766
1793
  provider,
1767
- search_plugins[0].provider,
1768
1794
  )
1769
1795
  else:
1770
1796
  provider_plugin = list(
@@ -1772,11 +1798,6 @@ class EODataAccessGateway:
1772
1798
  )[0]
1773
1799
  search_plugins.remove(provider_plugin)
1774
1800
  search_plugins.insert(0, provider_plugin)
1775
- logger.info(
1776
- "Searching product type '%s' on provider: %s",
1777
- product_type,
1778
- search_plugins[0].provider,
1779
- )
1780
1801
  # Add product_types_config to plugin config. This dict contains product
1781
1802
  # type metadata that will also be stored in each product's properties.
1782
1803
  for search_plugin in search_plugins:
@@ -1814,17 +1835,13 @@ class EODataAccessGateway:
1814
1835
  """Internal method that performs a search on a given provider.
1815
1836
 
1816
1837
  :param search_plugin: A search plugin
1817
- :type search_plugin: eodag.plugins.base.Search
1818
1838
  :param count: (optional) Whether to run a query with a count request or not
1819
- :type count: bool
1820
1839
  :param raise_errors: (optional) When an error occurs when searching, if this is set to
1821
1840
  True, the error is raised
1822
- :type raise_errors: bool
1823
1841
  :param kwargs: Some other criteria that will be used to do the search
1824
- :type kwargs: Any
1825
1842
  :returns: A collection of EO products matching the criteria
1826
- :rtype: tuple(:class:`~eodag.api.search_result.SearchResult`, int or None)
1827
1843
  """
1844
+ logger.info("Searching on provider %s", search_plugin.provider)
1828
1845
  max_items_per_page = getattr(search_plugin.config, "pagination", {}).get(
1829
1846
  "max_items_per_page", DEFAULT_MAX_ITEMS_PER_PAGE
1830
1847
  )
@@ -1842,19 +1859,23 @@ class EODataAccessGateway:
1842
1859
  max_items_per_page,
1843
1860
  )
1844
1861
 
1845
- need_auth = getattr(search_plugin.config, "need_auth", False)
1846
- auth_plugin = self._plugins_manager.get_auth_plugin(search_plugin.provider)
1847
- can_authenticate = callable(getattr(auth_plugin, "authenticate", None))
1848
-
1849
1862
  results: List[EOProduct] = []
1850
1863
  total_results: Optional[int] = 0 if count else None
1851
1864
 
1865
+ errors: List[Tuple[str, Exception]] = []
1866
+
1852
1867
  try:
1853
1868
  prep = PreparedSearch(count=count)
1854
- if need_auth and auth_plugin and can_authenticate:
1855
- prep.auth = auth_plugin.authenticate()
1856
1869
 
1857
- prep.auth_plugin = auth_plugin
1870
+ # append auth if needed
1871
+ if getattr(search_plugin.config, "need_auth", False):
1872
+ if auth := self._plugins_manager.get_auth(
1873
+ search_plugin.provider,
1874
+ getattr(search_plugin.config, "api_endpoint", None),
1875
+ search_plugin.config,
1876
+ ):
1877
+ prep.auth = auth
1878
+
1858
1879
  prep.page = kwargs.pop("page", None)
1859
1880
  prep.items_per_page = kwargs.pop("items_per_page", None)
1860
1881
 
@@ -1881,6 +1902,7 @@ class EODataAccessGateway:
1881
1902
  pattern = re.compile(r"[^\w,]+")
1882
1903
  try:
1883
1904
  guesses = self.guess_product_type(
1905
+ intersect=False,
1884
1906
  **{
1885
1907
  k: pattern.sub("", str(v).upper())
1886
1908
  for k, v in eo_product.properties.items()
@@ -1894,7 +1916,7 @@ class EODataAccessGateway:
1894
1916
  "keywords",
1895
1917
  ]
1896
1918
  and v is not None
1897
- }
1919
+ },
1898
1920
  )
1899
1921
  except NoMatchingProductType:
1900
1922
  pass
@@ -1907,12 +1929,31 @@ class EODataAccessGateway:
1907
1929
  eo_product.product_type
1908
1930
  )
1909
1931
  except NoMatchingProductType:
1910
- logger.warning("product type %s not found", eo_product.product_type)
1932
+ logger.debug("product type %s not found", eo_product.product_type)
1911
1933
 
1912
1934
  if eo_product.search_intersection is not None:
1913
1935
  download_plugin = self._plugins_manager.get_download_plugin(
1914
1936
  eo_product
1915
1937
  )
1938
+ if len(eo_product.assets) > 0:
1939
+ matching_url = next(iter(eo_product.assets.values()))["href"]
1940
+ elif eo_product.properties.get("storageStatus") != ONLINE_STATUS:
1941
+ matching_url = eo_product.properties.get(
1942
+ "orderLink"
1943
+ ) or eo_product.properties.get("downloadLink")
1944
+ else:
1945
+ matching_url = eo_product.properties.get("downloadLink")
1946
+
1947
+ try:
1948
+ auth_plugin = next(
1949
+ self._plugins_manager.get_auth_plugins(
1950
+ search_plugin.provider,
1951
+ matching_url=matching_url,
1952
+ matching_conf=download_plugin.config,
1953
+ )
1954
+ )
1955
+ except StopIteration:
1956
+ auth_plugin = None
1916
1957
  eo_product.register_downloader(download_plugin, auth_plugin)
1917
1958
 
1918
1959
  results.extend(res)
@@ -1942,13 +1983,6 @@ class EODataAccessGateway:
1942
1983
  "the total number of products matching the search criteria"
1943
1984
  )
1944
1985
  except Exception as e:
1945
- log_msg = f"No result from provider '{search_plugin.provider}' due to an error during search."
1946
- if not raise_errors:
1947
- log_msg += " Raise verbosity of log messages for details"
1948
- logger.info(log_msg)
1949
- # keep only the message from exception args
1950
- if len(e.args) > 1:
1951
- e.args = (e.args[0],)
1952
1986
  if raise_errors:
1953
1987
  # Raise the error, letting the application wrapping eodag know that
1954
1988
  # something went bad. This way it will be able to decide what to do next
@@ -1958,16 +1992,14 @@ class EODataAccessGateway:
1958
1992
  "Error while searching on provider %s (ignored):",
1959
1993
  search_plugin.provider,
1960
1994
  )
1961
- self.search_errors.add((search_plugin.provider, e))
1962
- return SearchResult(results, total_results)
1995
+ errors.append((search_plugin.provider, e))
1996
+ return SearchResult(results, total_results, errors)
1963
1997
 
1964
1998
  def crunch(self, results: SearchResult, **kwargs: Any) -> SearchResult:
1965
1999
  """Apply the filters given through the keyword arguments to the results
1966
2000
 
1967
2001
  :param results: The results of a eodag search request
1968
- :type results: :class:`~eodag.api.search_result.SearchResult`
1969
2002
  :returns: The result of successively applying all the filters to the results
1970
- :rtype: :class:`~eodag.api.search_result.SearchResult`
1971
2003
  """
1972
2004
  search_criteria = kwargs.pop("search_criteria", {})
1973
2005
  for cruncher_name, cruncher_args in kwargs.items():
@@ -1983,7 +2015,6 @@ class EODataAccessGateway:
1983
2015
  by extent (i.e. bounding box).
1984
2016
 
1985
2017
  :param searches: List of eodag SearchResult
1986
- :type searches: list
1987
2018
  :returns: list of :class:`~eodag.api.search_result.SearchResult`
1988
2019
  """
1989
2020
  # Dict with extents as keys, each extent being defined by a str
@@ -2014,33 +2045,32 @@ class EODataAccessGateway:
2014
2045
  """Download all products resulting from a search.
2015
2046
 
2016
2047
  :param search_result: A collection of EO products resulting from a search
2017
- :type search_result: :class:`~eodag.api.search_result.SearchResult`
2018
2048
  :param downloaded_callback: (optional) A method or a callable object which takes
2019
2049
  as parameter the ``product``. You can use the base class
2020
- :class:`~eodag.api.product.DownloadedCallback` and override
2050
+ :class:`~eodag.utils.DownloadedCallback` and override
2021
2051
  its ``__call__`` method. Will be called each time a product
2022
2052
  finishes downloading
2023
- :type downloaded_callback: Callable[[:class:`~eodag.api.product._product.EOProduct`], None]
2024
- or None
2025
2053
  :param progress_callback: (optional) A method or a callable object
2026
2054
  which takes a current size and a maximum
2027
2055
  size as inputs and handle progress bar
2028
2056
  creation and update to give the user a
2029
2057
  feedback on the download progress
2030
- :type progress_callback: :class:`~eodag.utils.ProgressCallback` or None
2031
2058
  :param wait: (optional) If download fails, wait time in minutes between
2032
2059
  two download tries of the same product
2033
- :type wait: int
2034
2060
  :param timeout: (optional) If download fails, maximum time in minutes
2035
2061
  before stop retrying to download
2036
- :type timeout: int
2037
- :param kwargs: `outputs_prefix` (str), `extract` (bool), `delete_archive` (bool)
2038
- and `dl_url_params` (dict) can be provided as additional kwargs
2039
- and will override any other values defined in a configuration
2040
- file or with environment variables.
2041
- :type kwargs: Union[str, bool, dict]
2062
+ :param kwargs: Additional keyword arguments from the download plugin configuration class that can
2063
+ be provided to override any other values defined in a configuration file
2064
+ or with environment variables:
2065
+
2066
+ * ``output_dir`` - where to store downloaded products, as an absolute file path
2067
+ (Default: local temporary directory)
2068
+ * ``output_extension`` - downloaded file extension
2069
+ * ``extract`` - whether to extract the downloaded products, only applies to archived products
2070
+ * ``dl_url_params`` - additional parameters to pass over to the download url as an url parameter
2071
+ * ``delete_archive`` - whether to delete the downloaded archives
2072
+ * ``asset`` - regex filter to identify assets to download
2042
2073
  :returns: A collection of the absolute paths to the downloaded products
2043
- :rtype: list
2044
2074
  """
2045
2075
  paths = []
2046
2076
  if search_result:
@@ -2069,11 +2099,8 @@ class EODataAccessGateway:
2069
2099
  """Registers results of a search into a geojson file.
2070
2100
 
2071
2101
  :param search_result: A collection of EO products resulting from a search
2072
- :type search_result: :class:`~eodag.api.search_result.SearchResult`
2073
2102
  :param filename: (optional) The name of the file to generate
2074
- :type filename: str
2075
2103
  :returns: The name of the created file
2076
- :rtype: str
2077
2104
  """
2078
2105
  with open(filename, "w") as fh:
2079
2106
  geojson.dump(search_result, fh)
@@ -2084,9 +2111,7 @@ class EODataAccessGateway:
2084
2111
  """Loads results of a search from a geojson file.
2085
2112
 
2086
2113
  :param filename: A filename containing a search result encoded as a geojson
2087
- :type filename: str
2088
2114
  :returns: The search results encoded in `filename`
2089
- :rtype: :class:`~eodag.api.search_result.SearchResult`
2090
2115
  """
2091
2116
  with open(filename, "r") as fh:
2092
2117
  return SearchResult.from_geojson(geojson.load(fh))
@@ -2096,19 +2121,17 @@ class EODataAccessGateway:
2096
2121
  products with the information needed to download itself
2097
2122
 
2098
2123
  :param filename: A filename containing a search result encoded as a geojson
2099
- :type filename: str
2100
2124
  :returns: The search results encoded in `filename`
2101
- :rtype: :class:`~eodag.api.search_result.SearchResult`
2102
2125
  """
2103
2126
  products = self.deserialize(filename)
2104
2127
  for i, product in enumerate(products):
2105
2128
  if product.downloader is None:
2129
+ downloader = self._plugins_manager.get_download_plugin(product)
2106
2130
  auth = product.downloader_auth
2107
2131
  if auth is None:
2108
- auth = self._plugins_manager.get_auth_plugin(product.provider)
2109
- products[i].register_downloader(
2110
- self._plugins_manager.get_download_plugin(product), auth
2111
- )
2132
+ auth = self._plugins_manager.get_auth_plugin(downloader, product)
2133
+ products[i].register_downloader(downloader, auth)
2134
+
2112
2135
  return products
2113
2136
 
2114
2137
  @_deprecated(
@@ -2131,22 +2154,14 @@ class EODataAccessGateway:
2131
2154
  the response content to an API request.
2132
2155
 
2133
2156
  :param filename: A filename containing features encoded as a geojson
2134
- :type filename: str
2135
2157
  :param recursive: (optional) Browse recursively in child nodes if True
2136
- :type recursive: bool
2137
- :param max_connections: (optional) Maximum number of connections for HTTP requests
2138
- :type max_connections: int
2158
+ :param max_connections: (optional) Maximum number of connections for concurrent HTTP requests
2139
2159
  :param provider: (optional) Data provider
2140
- :type provider: str
2141
2160
  :param productType: (optional) Data product type
2142
- :type productType: str
2143
2161
  :param timeout: (optional) Timeout in seconds for each internal HTTP request
2144
- :type timeout: float
2145
2162
  :param kwargs: Parameters that will be stored in the result as
2146
2163
  search criteria
2147
- :type kwargs: Any
2148
2164
  :returns: The search results encoded in `filename`
2149
- :rtype: :class:`~eodag.api.search_result.SearchResult`
2150
2165
 
2151
2166
  .. deprecated:: 2.2.1
2152
2167
  Use the :class:`~eodag.plugins.search.static_stac_search.StaticStacSearch` search plugin instead.
@@ -2211,26 +2226,27 @@ class EODataAccessGateway:
2211
2226
  trying to download the product.
2212
2227
 
2213
2228
  :param product: The EO product to download
2214
- :type product: :class:`~eodag.api.product._product.EOProduct`
2215
2229
  :param progress_callback: (optional) A method or a callable object
2216
2230
  which takes a current size and a maximum
2217
2231
  size as inputs and handle progress bar
2218
2232
  creation and update to give the user a
2219
2233
  feedback on the download progress
2220
- :type progress_callback: :class:`~eodag.utils.ProgressCallback` or None
2221
2234
  :param wait: (optional) If download fails, wait time in minutes between
2222
2235
  two download tries
2223
- :type wait: int
2224
2236
  :param timeout: (optional) If download fails, maximum time in minutes
2225
2237
  before stop retrying to download
2226
- :type timeout: int
2227
- :param kwargs: `outputs_prefix` (str), `extract` (bool), `delete_archive` (bool)
2228
- and `dl_url_params` (dict) can be provided as additional kwargs
2229
- and will override any other values defined in a configuration
2230
- file or with environment variables.
2231
- :type kwargs: Union[str, bool, dict]
2238
+ :param kwargs: Additional keyword arguments from the download plugin configuration class that can
2239
+ be provided to override any other values defined in a configuration file
2240
+ or with environment variables:
2241
+
2242
+ * ``output_dir`` - where to store downloaded products, as an absolute file path
2243
+ (Default: local temporary directory)
2244
+ * ``output_extension`` - downloaded file extension
2245
+ * ``extract`` - whether to extract the downloaded products, only applies to archived products
2246
+ * ``dl_url_params`` - additional parameters to pass over to the download url as an url parameter
2247
+ * ``delete_archive`` - whether to delete the downloaded archives
2248
+ * ``asset`` - regex filter to identify assets to download
2232
2249
  :returns: The absolute path to the downloaded product in the local filesystem
2233
- :rtype: str
2234
2250
  :raises: :class:`~eodag.utils.exceptions.PluginImplementationError`
2235
2251
  :raises: :class:`RuntimeError`
2236
2252
  """
@@ -2246,22 +2262,18 @@ class EODataAccessGateway:
2246
2262
 
2247
2263
  def _setup_downloader(self, product: EOProduct) -> None:
2248
2264
  if product.downloader is None:
2265
+ downloader = self._plugins_manager.get_download_plugin(product)
2249
2266
  auth = product.downloader_auth
2250
2267
  if auth is None:
2251
- auth = self._plugins_manager.get_auth_plugin(product.provider)
2252
- product.register_downloader(
2253
- self._plugins_manager.get_download_plugin(product), auth
2254
- )
2268
+ auth = self._plugins_manager.get_auth_plugin(downloader, product)
2269
+ product.register_downloader(downloader, auth)
2255
2270
 
2256
2271
  def get_cruncher(self, name: str, **options: Any) -> Crunch:
2257
2272
  """Build a crunch plugin from a configuration
2258
2273
 
2259
2274
  :param name: The name of the cruncher to build
2260
- :type name: str
2261
2275
  :param options: The configuration options of the cruncher
2262
- :type options: dict
2263
2276
  :returns: The cruncher named ``name``
2264
- :rtype: :class:`~eodag.plugins.crunch.Crunch`
2265
2277
  """
2266
2278
  plugin_conf = {"name": name}
2267
2279
  plugin_conf.update({key.replace("-", "_"): val for key, val in options.items()})
@@ -2273,17 +2285,14 @@ class EODataAccessGateway:
2273
2285
  """Fetch the queryable properties for a given product type and/or provider.
2274
2286
 
2275
2287
  :param provider: (optional) The provider.
2276
- :type provider: str
2277
2288
  :param kwargs: additional filters for queryables (`productType` or other search
2278
2289
  arguments)
2279
- :type kwargs: Any
2280
2290
 
2281
2291
  :raises UnsupportedProductType: If the specified product type is not available for the
2282
2292
  provider.
2283
2293
 
2284
2294
  :returns: A dict containing the EODAG queryable properties, associating
2285
2295
  parameters to their annotated type
2286
- :rtype: Dict[str, Annotated[Any, FieldInfo]]
2287
2296
  """
2288
2297
  available_product_types = [
2289
2298
  pt["ID"]
@@ -2309,7 +2318,7 @@ class EODataAccessGateway:
2309
2318
 
2310
2319
  for plugin in self._plugins_manager.get_search_plugins(product_type, provider):
2311
2320
  if getattr(plugin.config, "need_auth", False) and (
2312
- auth := self._plugins_manager.get_auth_plugin(plugin.provider)
2321
+ auth := self._plugins_manager.get_auth_plugin(plugin)
2313
2322
  ):
2314
2323
  plugin.auth = auth.authenticate()
2315
2324
  providers_queryables[plugin.provider] = plugin.list_queryables(
@@ -2339,9 +2348,8 @@ class EODataAccessGateway:
2339
2348
  """For each provider, gives its available sortable parameter(s) and its maximum
2340
2349
  number of them if it supports the sorting feature, otherwise gives None.
2341
2350
 
2342
- :returns: A dictionnary with providers as keys and dictionnary of sortable parameter(s) and
2351
+ :returns: A dictionary with providers as keys and dictionary of sortable parameter(s) and
2343
2352
  its (their) maximum number as value(s).
2344
- :rtype: dict
2345
2353
  :raises: :class:`~eodag.utils.exceptions.UnsupportedProvider`
2346
2354
  """
2347
2355
  sortables: Dict[str, Optional[ProviderSortables]] = {}