eodag 2.12.0__py3-none-any.whl → 3.0.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (93) hide show
  1. eodag/__init__.py +6 -8
  2. eodag/api/core.py +654 -538
  3. eodag/api/product/__init__.py +12 -2
  4. eodag/api/product/_assets.py +59 -16
  5. eodag/api/product/_product.py +100 -93
  6. eodag/api/product/drivers/__init__.py +7 -2
  7. eodag/api/product/drivers/base.py +0 -3
  8. eodag/api/product/metadata_mapping.py +192 -96
  9. eodag/api/search_result.py +69 -10
  10. eodag/cli.py +55 -25
  11. eodag/config.py +391 -116
  12. eodag/plugins/apis/base.py +11 -165
  13. eodag/plugins/apis/ecmwf.py +36 -25
  14. eodag/plugins/apis/usgs.py +80 -35
  15. eodag/plugins/authentication/aws_auth.py +13 -4
  16. eodag/plugins/authentication/base.py +10 -1
  17. eodag/plugins/authentication/generic.py +2 -2
  18. eodag/plugins/authentication/header.py +31 -6
  19. eodag/plugins/authentication/keycloak.py +17 -84
  20. eodag/plugins/authentication/oauth.py +3 -3
  21. eodag/plugins/authentication/openid_connect.py +268 -49
  22. eodag/plugins/authentication/qsauth.py +4 -1
  23. eodag/plugins/authentication/sas_auth.py +9 -2
  24. eodag/plugins/authentication/token.py +98 -47
  25. eodag/plugins/authentication/token_exchange.py +122 -0
  26. eodag/plugins/crunch/base.py +3 -1
  27. eodag/plugins/crunch/filter_date.py +3 -9
  28. eodag/plugins/crunch/filter_latest_intersect.py +0 -3
  29. eodag/plugins/crunch/filter_latest_tpl_name.py +1 -4
  30. eodag/plugins/crunch/filter_overlap.py +4 -8
  31. eodag/plugins/crunch/filter_property.py +5 -11
  32. eodag/plugins/download/aws.py +149 -185
  33. eodag/plugins/download/base.py +88 -97
  34. eodag/plugins/download/creodias_s3.py +1 -1
  35. eodag/plugins/download/http.py +638 -310
  36. eodag/plugins/download/s3rest.py +47 -45
  37. eodag/plugins/manager.py +228 -88
  38. eodag/plugins/search/__init__.py +36 -0
  39. eodag/plugins/search/base.py +239 -30
  40. eodag/plugins/search/build_search_result.py +382 -37
  41. eodag/plugins/search/cop_marine.py +441 -0
  42. eodag/plugins/search/creodias_s3.py +25 -20
  43. eodag/plugins/search/csw.py +5 -7
  44. eodag/plugins/search/data_request_search.py +61 -30
  45. eodag/plugins/search/qssearch.py +713 -255
  46. eodag/plugins/search/static_stac_search.py +106 -40
  47. eodag/resources/ext_product_types.json +1 -1
  48. eodag/resources/product_types.yml +1921 -34
  49. eodag/resources/providers.yml +4091 -3655
  50. eodag/resources/stac.yml +50 -216
  51. eodag/resources/stac_api.yml +71 -25
  52. eodag/resources/stac_provider.yml +5 -0
  53. eodag/resources/user_conf_template.yml +89 -32
  54. eodag/rest/__init__.py +6 -0
  55. eodag/rest/cache.py +70 -0
  56. eodag/rest/config.py +68 -0
  57. eodag/rest/constants.py +26 -0
  58. eodag/rest/core.py +735 -0
  59. eodag/rest/errors.py +178 -0
  60. eodag/rest/server.py +264 -431
  61. eodag/rest/stac.py +442 -836
  62. eodag/rest/types/collections_search.py +44 -0
  63. eodag/rest/types/eodag_search.py +238 -47
  64. eodag/rest/types/queryables.py +164 -0
  65. eodag/rest/types/stac_search.py +273 -0
  66. eodag/rest/utils/__init__.py +216 -0
  67. eodag/rest/utils/cql_evaluate.py +119 -0
  68. eodag/rest/utils/rfc3339.py +64 -0
  69. eodag/types/__init__.py +106 -10
  70. eodag/types/bbox.py +15 -14
  71. eodag/types/download_args.py +40 -0
  72. eodag/types/search_args.py +57 -7
  73. eodag/types/whoosh.py +79 -0
  74. eodag/utils/__init__.py +110 -91
  75. eodag/utils/constraints.py +37 -45
  76. eodag/utils/exceptions.py +39 -22
  77. eodag/utils/import_system.py +0 -4
  78. eodag/utils/logging.py +37 -80
  79. eodag/utils/notebook.py +4 -4
  80. eodag/utils/repr.py +113 -0
  81. eodag/utils/requests.py +128 -0
  82. eodag/utils/rest.py +100 -0
  83. eodag/utils/stac_reader.py +93 -21
  84. {eodag-2.12.0.dist-info → eodag-3.0.0.dist-info}/METADATA +88 -53
  85. eodag-3.0.0.dist-info/RECORD +109 -0
  86. {eodag-2.12.0.dist-info → eodag-3.0.0.dist-info}/WHEEL +1 -1
  87. {eodag-2.12.0.dist-info → eodag-3.0.0.dist-info}/entry_points.txt +7 -5
  88. eodag/plugins/apis/cds.py +0 -540
  89. eodag/rest/types/stac_queryables.py +0 -134
  90. eodag/rest/utils.py +0 -1133
  91. eodag-2.12.0.dist-info/RECORD +0 -94
  92. {eodag-2.12.0.dist-info → eodag-3.0.0.dist-info}/LICENSE +0 -0
  93. {eodag-2.12.0.dist-info → eodag-3.0.0.dist-info}/top_level.txt +0 -0
eodag/api/core.py CHANGED
@@ -17,24 +17,14 @@
17
17
  # limitations under the License.
18
18
  from __future__ import annotations
19
19
 
20
+ import datetime
20
21
  import logging
21
22
  import os
22
23
  import re
23
24
  import shutil
24
25
  import tempfile
25
26
  from operator import itemgetter
26
- from typing import (
27
- TYPE_CHECKING,
28
- AbstractSet,
29
- Any,
30
- Dict,
31
- Iterator,
32
- List,
33
- Optional,
34
- Set,
35
- Tuple,
36
- Union,
37
- )
27
+ from typing import TYPE_CHECKING, Any, Dict, Iterator, List, Optional, Set, Tuple, Union
38
28
 
39
29
  import geojson
40
30
  import pkg_resources
@@ -47,12 +37,15 @@ from whoosh.index import create_in, exists_in, open_dir
47
37
  from whoosh.qparser import QueryParser
48
38
 
49
39
  from eodag.api.product.metadata_mapping import (
50
- NOT_MAPPED,
40
+ ONLINE_STATUS,
51
41
  mtd_cfg_as_conversion_and_querypath,
52
42
  )
53
43
  from eodag.api.search_result import SearchResult
54
44
  from eodag.config import (
45
+ PLUGINS_TOPICS_KEYS,
46
+ PluginConfig,
55
47
  SimpleYamlProxyConfig,
48
+ credentials_in_auth,
56
49
  get_ext_product_types_conf,
57
50
  load_default_config,
58
51
  load_stac_provider_config,
@@ -61,11 +54,15 @@ from eodag.config import (
61
54
  override_config_from_file,
62
55
  override_config_from_mapping,
63
56
  provider_config_init,
57
+ share_credentials,
64
58
  )
65
59
  from eodag.plugins.manager import PluginManager
60
+ from eodag.plugins.search import PreparedSearch
66
61
  from eodag.plugins.search.build_search_result import BuildPostSearchResult
62
+ from eodag.plugins.search.qssearch import PostJsonSearch
67
63
  from eodag.types import model_fields_to_annotated
68
- from eodag.types.queryables import CommonQueryables, Queryables
64
+ from eodag.types.queryables import CommonQueryables
65
+ from eodag.types.whoosh import EODAGQueryParser
69
66
  from eodag.utils import (
70
67
  DEFAULT_DOWNLOAD_TIMEOUT,
71
68
  DEFAULT_DOWNLOAD_WAIT,
@@ -77,23 +74,22 @@ from eodag.utils import (
77
74
  MockResponse,
78
75
  _deprecated,
79
76
  copy_deepcopy,
80
- deepcopy,
81
- get_args,
82
77
  get_geometry_from_various,
83
78
  makedirs,
84
79
  obj_md5sum,
80
+ sort_dict,
85
81
  string_to_jsonpath,
86
82
  uri_to_path,
87
83
  )
88
84
  from eodag.utils.exceptions import (
89
- AuthenticationError,
90
- MisconfiguredError,
85
+ EodagError,
91
86
  NoMatchingProductType,
92
87
  PluginImplementationError,
93
88
  RequestError,
94
89
  UnsupportedProductType,
95
90
  UnsupportedProvider,
96
91
  )
92
+ from eodag.utils.rest import rfc3339_str_to_datetime
97
93
  from eodag.utils.stac_reader import fetch_stac_items
98
94
 
99
95
  if TYPE_CHECKING:
@@ -104,7 +100,9 @@ if TYPE_CHECKING:
104
100
  from eodag.plugins.apis.base import Api
105
101
  from eodag.plugins.crunch.base import Crunch
106
102
  from eodag.plugins.search.base import Search
107
- from eodag.utils import Annotated, DownloadedCallback, ProgressCallback
103
+ from eodag.types import ProviderSortables
104
+ from eodag.types.download_args import DownloadConf
105
+ from eodag.utils import Annotated, DownloadedCallback, ProgressCallback, Unpack
108
106
 
109
107
  logger = logging.getLogger("eodag.core")
110
108
 
@@ -114,9 +112,7 @@ class EODataAccessGateway:
114
112
  from different types of providers.
115
113
 
116
114
  :param user_conf_file_path: (optional) Path to the user configuration file
117
- :type user_conf_file_path: str
118
115
  :param locations_conf_path: (optional) Path to the locations configuration file
119
- :type locations_conf_path: str
120
116
  """
121
117
 
122
118
  def __init__(
@@ -176,10 +172,15 @@ class EODataAccessGateway:
176
172
  # Second level override: From environment variables
177
173
  override_config_from_env(self.providers_config)
178
174
 
175
+ # share credentials between updated plugins confs
176
+ share_credentials(self.providers_config)
177
+
179
178
  # init updated providers conf
180
- stac_provider_config = load_stac_provider_config()
181
179
  for provider in self.providers_config.keys():
182
- provider_config_init(self.providers_config[provider], stac_provider_config)
180
+ provider_config_init(
181
+ self.providers_config[provider],
182
+ load_stac_provider_config(),
183
+ )
183
184
 
184
185
  # re-build _plugins_manager using up-to-date providers_config
185
186
  self._plugins_manager.rebuild(self.providers_config)
@@ -225,7 +226,6 @@ class EODataAccessGateway:
225
226
  os.path.join(self.conf_dir, "shp"),
226
227
  )
227
228
  self.set_locations_conf(locations_conf_path)
228
- self.search_errors: Set = set()
229
229
 
230
230
  def get_version(self) -> str:
231
231
  """Get eodag package version"""
@@ -247,7 +247,6 @@ class EODataAccessGateway:
247
247
  if "unsupported pickle protocol" in str(ve):
248
248
  logger.debug("Need to recreate whoosh .index: '%s'", ve)
249
249
  create_index = True
250
- shutil.rmtree(index_dir)
251
250
  # Unexpected error
252
251
  else:
253
252
  logger.error(
@@ -261,13 +260,14 @@ class EODataAccessGateway:
261
260
  if self._product_types_index is None:
262
261
  logger.debug("Opening product types index in %s", index_dir)
263
262
  self._product_types_index = open_dir(index_dir)
264
- try:
265
- self.guess_product_type(md5=self.product_types_config_md5)
266
- except NoMatchingProductType:
267
- create_index = True
268
- finally:
269
- if create_index:
270
- shutil.rmtree(index_dir)
263
+
264
+ with self._product_types_index.searcher() as searcher:
265
+ p = QueryParser("md5", self._product_types_index.schema, plugins=[])
266
+ query = p.parse(self.product_types_config_md5)
267
+ results = searcher.search(query, limit=1)
268
+
269
+ if not results:
270
+ create_index = True
271
271
  logger.debug(
272
272
  "Out-of-date product types index removed from %s", index_dir
273
273
  )
@@ -284,9 +284,8 @@ class EODataAccessGateway:
284
284
  )
285
285
 
286
286
  product_types_schema = Schema(
287
- ID=fields.STORED,
288
- alias=fields.ID,
289
- abstract=fields.STORED,
287
+ ID=fields.ID(stored=True),
288
+ abstract=fields.TEXT,
290
289
  instrument=fields.IDLIST,
291
290
  platform=fields.ID,
292
291
  platformSerialIdentifier=fields.IDLIST,
@@ -294,10 +293,11 @@ class EODataAccessGateway:
294
293
  sensorType=fields.ID,
295
294
  md5=fields.ID,
296
295
  license=fields.ID,
297
- title=fields.ID,
298
- missionStartDate=fields.ID,
299
- missionEndDate=fields.ID,
296
+ title=fields.TEXT,
297
+ missionStartDate=fields.STORED,
298
+ missionEndDate=fields.STORED,
300
299
  keywords=fields.KEYWORD(analyzer=kw_analyzer),
300
+ stacCollection=fields.STORED,
301
301
  )
302
302
  self._product_types_index = create_in(index_dir, product_types_schema)
303
303
  ix_writer = self._product_types_index.writer()
@@ -320,7 +320,6 @@ class EODataAccessGateway:
320
320
 
321
321
  :param provider: The name of the provider that should be considered as the
322
322
  preferred provider to be used for this instance
323
- :type provider: str
324
323
  """
325
324
  if provider not in self.available_providers():
326
325
  raise UnsupportedProvider(
@@ -336,7 +335,6 @@ class EODataAccessGateway:
336
335
  products, along with its priority.
337
336
 
338
337
  :returns: The provider with the maximum priority and its priority
339
- :rtype: tuple(str, int)
340
338
  """
341
339
  providers_with_priority = [
342
340
  (provider, conf.priority)
@@ -345,15 +343,24 @@ class EODataAccessGateway:
345
343
  preferred, priority = max(providers_with_priority, key=itemgetter(1))
346
344
  return preferred, priority
347
345
 
348
- def update_providers_config(self, yaml_conf: str) -> None:
346
+ def update_providers_config(
347
+ self,
348
+ yaml_conf: Optional[str] = None,
349
+ dict_conf: Optional[Dict[str, Any]] = None,
350
+ ) -> None:
349
351
  """Update providers configuration with given input.
350
352
  Can be used to add a provider to existing configuration or update
351
353
  an existing one.
352
354
 
353
355
  :param yaml_conf: YAML formated provider configuration
354
- :type yaml_conf: str
356
+ :param dict_conf: provider configuration as dictionary in place of ``yaml_conf``
355
357
  """
356
- conf_update = yaml.safe_load(yaml_conf)
358
+ if dict_conf is not None:
359
+ conf_update = dict_conf
360
+ elif yaml_conf is not None:
361
+ conf_update = yaml.safe_load(yaml_conf)
362
+ else:
363
+ return None
357
364
 
358
365
  # restore the pruned configuration
359
366
  for provider in list(self._pruned_providers_config.keys()):
@@ -366,60 +373,108 @@ class EODataAccessGateway:
366
373
  provider
367
374
  )
368
375
 
369
- # check if metada-mapping as already been built as jsonpath in providers_config
370
- for provider, provider_conf in conf_update.items():
371
- if (
372
- provider in self.providers_config
373
- and "metadata_mapping" in provider_conf.get("search", {})
374
- ):
375
- search_plugin_key = "search"
376
- elif (
377
- provider in self.providers_config
378
- and "metadata_mapping" in provider_conf.get("api", {})
379
- ):
380
- search_plugin_key = "api"
381
- else:
382
- continue
383
- # get some already configured value
384
- configured_metadata_mapping = getattr(
385
- self.providers_config[provider], search_plugin_key
386
- ).metadata_mapping
387
- some_configured_value = next(iter(configured_metadata_mapping.values()))
388
- # check if the configured value has already been built as jsonpath
389
- if (
390
- isinstance(some_configured_value, list)
391
- and isinstance(some_configured_value[1], tuple)
392
- or isinstance(some_configured_value, tuple)
393
- ):
394
- # also build as jsonpath the incoming conf
395
- mtd_cfg_as_conversion_and_querypath(
396
- deepcopy(
397
- conf_update[provider][search_plugin_key]["metadata_mapping"]
398
- ),
399
- conf_update[provider][search_plugin_key]["metadata_mapping"],
400
- )
401
-
402
376
  override_config_from_mapping(self.providers_config, conf_update)
403
377
 
404
- stac_provider_config = load_stac_provider_config()
378
+ # share credentials between updated plugins confs
379
+ share_credentials(self.providers_config)
380
+
405
381
  for provider in conf_update.keys():
406
- provider_config_init(self.providers_config[provider], stac_provider_config)
382
+ provider_config_init(
383
+ self.providers_config[provider],
384
+ load_stac_provider_config(),
385
+ )
386
+ setattr(self.providers_config[provider], "product_types_fetched", False)
407
387
  # re-create _plugins_manager using up-to-date providers_config
408
388
  self._plugins_manager.build_product_type_to_provider_config_map()
409
389
 
390
+ def add_provider(
391
+ self,
392
+ name: str,
393
+ url: Optional[str] = None,
394
+ priority: Optional[int] = None,
395
+ search: Dict[str, Any] = {"type": "StacSearch"},
396
+ products: Dict[str, Any] = {
397
+ GENERIC_PRODUCT_TYPE: {"productType": "{productType}"}
398
+ },
399
+ download: Dict[str, Any] = {"type": "HTTPDownload", "auth_error_code": 401},
400
+ **kwargs: Dict[str, Any],
401
+ ):
402
+ """Adds a new provider.
403
+
404
+ ``search``, ``products`` & ``download`` already have default values that will be
405
+ updated (not replaced), with user provided ones:
406
+
407
+ * ``search`` : ``{"type": "StacSearch"}``
408
+ * ``products`` : ``{"GENERIC_PRODUCT_TYPE": {"productType": "{productType}"}}``
409
+ * ``download`` : ``{"type": "HTTPDownload", "auth_error_code": 401}``
410
+
411
+ :param name: Name of provider
412
+ :param url: Provider url, also used as ``search["api_endpoint"]`` if not defined
413
+ :param priority: Provider priority. If None, provider will be set as preferred (highest priority)
414
+ :param search: Search :class:`~eodag.config.PluginConfig` mapping
415
+ :param products: Provider product types mapping
416
+ :param download: Download :class:`~eodag.config.PluginConfig` mapping
417
+ :param kwargs: Additional :class:`~eodag.config.ProviderConfig` mapping
418
+ """
419
+ conf_dict: Dict[str, Any] = {
420
+ name: {
421
+ "url": url,
422
+ "search": {"type": "StacSearch", **search},
423
+ "products": {
424
+ GENERIC_PRODUCT_TYPE: {"productType": "{productType}"},
425
+ **products,
426
+ },
427
+ "download": {
428
+ "type": "HTTPDownload",
429
+ "auth_error_code": 401,
430
+ **download,
431
+ },
432
+ **kwargs,
433
+ }
434
+ }
435
+ if priority is not None:
436
+ conf_dict[name]["priority"] = priority
437
+ # if provided, use url as default search api_endpoint
438
+ if (
439
+ url
440
+ and conf_dict[name].get("search", {})
441
+ and not conf_dict[name]["search"].get("api_endpoint")
442
+ ):
443
+ conf_dict[name]["search"]["api_endpoint"] = url
444
+
445
+ # api plugin usage: remove unneeded search/download/auth plugin conf
446
+ if conf_dict[name].get("api"):
447
+ for k in PLUGINS_TOPICS_KEYS:
448
+ if k != "api":
449
+ conf_dict[name].pop(k, None)
450
+
451
+ self.update_providers_config(dict_conf=conf_dict)
452
+
453
+ if priority is None:
454
+ self.set_preferred_provider(name)
455
+
410
456
  def _prune_providers_list(self) -> None:
411
457
  """Removes from config providers needing auth that have no credentials set."""
412
458
  update_needed = False
413
459
  for provider in list(self.providers_config.keys()):
414
460
  conf = self.providers_config[provider]
415
461
 
416
- if hasattr(conf, "api") and getattr(conf.api, "need_auth", False):
417
- credentials_exist = any(
418
- [
419
- cred is not None
420
- for cred in getattr(conf.api, "credentials", {}).values()
421
- ]
462
+ # remove providers using skipped plugins
463
+ if [
464
+ v
465
+ for v in conf.__dict__.values()
466
+ if isinstance(v, PluginConfig)
467
+ and getattr(v, "type", None) in self._plugins_manager.skipped_plugins
468
+ ]:
469
+ self.providers_config.pop(provider)
470
+ logger.debug(
471
+ f"{provider}: provider needing unavailable plugin has been removed"
422
472
  )
473
+ continue
474
+
475
+ # check authentication
476
+ if hasattr(conf, "api") and getattr(conf.api, "need_auth", False):
477
+ credentials_exist = credentials_in_auth(conf.api)
423
478
  if not credentials_exist:
424
479
  # credentials needed but not found
425
480
  self._pruned_providers_config[provider] = self.providers_config.pop(
@@ -427,11 +482,11 @@ class EODataAccessGateway:
427
482
  )
428
483
  update_needed = True
429
484
  logger.info(
430
- "%s: provider needing auth for search has been pruned because no crendentials could be found",
485
+ "%s: provider needing auth for search has been pruned because no credentials could be found",
431
486
  provider,
432
487
  )
433
488
  elif hasattr(conf, "search") and getattr(conf.search, "need_auth", False):
434
- if not hasattr(conf, "auth"):
489
+ if not hasattr(conf, "auth") and not hasattr(conf, "search_auth"):
435
490
  # credentials needed but no auth plugin was found
436
491
  self._pruned_providers_config[provider] = self.providers_config.pop(
437
492
  provider
@@ -442,11 +497,13 @@ class EODataAccessGateway:
442
497
  provider,
443
498
  )
444
499
  continue
445
- credentials_exist = any(
446
- [
447
- cred is not None
448
- for cred in getattr(conf.auth, "credentials", {}).values()
449
- ]
500
+ credentials_exist = (
501
+ hasattr(conf, "search_auth")
502
+ and credentials_in_auth(conf.search_auth)
503
+ ) or (
504
+ not hasattr(conf, "search_auth")
505
+ and hasattr(conf, "auth")
506
+ and credentials_in_auth(conf.auth)
450
507
  )
451
508
  if not credentials_exist:
452
509
  # credentials needed but not found
@@ -455,7 +512,7 @@ class EODataAccessGateway:
455
512
  )
456
513
  update_needed = True
457
514
  logger.info(
458
- "%s: provider needing auth for search has been pruned because no crendentials could be found",
515
+ "%s: provider needing auth for search has been pruned because no credentials could be found",
459
516
  provider,
460
517
  )
461
518
  elif not hasattr(conf, "api") and not hasattr(conf, "search"):
@@ -495,16 +552,15 @@ class EODataAccessGateway:
495
552
  attr: FRA
496
553
 
497
554
  :param locations_conf_path: Path to the locations configuration file
498
- :type locations_conf_path: str
499
555
  """
500
556
  if os.path.isfile(locations_conf_path):
501
557
  locations_config = load_yml_config(locations_conf_path)
502
558
 
503
559
  main_key = next(iter(locations_config))
504
- locations_config = locations_config[main_key]
560
+ main_locations_config = locations_config[main_key]
505
561
 
506
562
  logger.info("Locations configuration loaded from %s" % locations_conf_path)
507
- self.locations_config: List[Dict[str, Any]] = locations_config
563
+ self.locations_config: List[Dict[str, Any]] = main_locations_config
508
564
  else:
509
565
  logger.info(
510
566
  "Could not load locations configuration from %s" % locations_conf_path
@@ -518,12 +574,9 @@ class EODataAccessGateway:
518
574
 
519
575
  :param provider: (optional) The name of a provider that must support the product
520
576
  types we are about to list
521
- :type provider: str
522
577
  :param fetch_providers: (optional) Whether to fetch providers for new product
523
578
  types or not
524
- :type fetch_providers: bool
525
579
  :returns: The list of the product types that can be accessed using eodag.
526
- :rtype: list(dict)
527
580
  :raises: :class:`~eodag.utils.exceptions.UnsupportedProvider`
528
581
  """
529
582
  if fetch_providers:
@@ -531,57 +584,66 @@ class EODataAccessGateway:
531
584
  self.fetch_product_types_list(provider=provider)
532
585
 
533
586
  product_types: List[Dict[str, Any]] = []
534
- if provider is not None:
535
- if provider in self.providers_config:
536
- provider_supported_products = self.providers_config[provider].products
537
- for product_type_id in provider_supported_products:
538
- if product_type_id == GENERIC_PRODUCT_TYPE:
539
- continue
540
- config = self.product_types_config[product_type_id]
541
- if "alias" in config:
542
- config["_id"] = product_type_id
543
- product_type_id = config["alias"]
544
- product_type = dict(ID=product_type_id, **config)
545
- if product_type_id not in product_types:
546
- product_types.append(product_type)
547
- return sorted(product_types, key=itemgetter("ID"))
587
+
588
+ providers_configs = (
589
+ list(self.providers_config.values())
590
+ if not provider
591
+ else [
592
+ p
593
+ for p in self.providers_config.values()
594
+ if provider in [p.name, getattr(p, "group", None)]
595
+ ]
596
+ )
597
+
598
+ if provider and not providers_configs:
548
599
  raise UnsupportedProvider(
549
600
  f"The requested provider is not (yet) supported: {provider}"
550
601
  )
551
- # Only get the product types supported by the available providers
552
- for provider in self.available_providers():
553
- current_product_type_ids = [pt["ID"] for pt in product_types]
554
- product_types.extend(
555
- [
556
- pt
557
- for pt in self.list_product_types(
558
- provider=provider, fetch_providers=False
559
- )
560
- if pt["ID"] not in current_product_type_ids
561
- ]
562
- )
602
+
603
+ for p in providers_configs:
604
+ for product_type_id in p.products: # type: ignore
605
+ if product_type_id == GENERIC_PRODUCT_TYPE:
606
+ continue
607
+ config = self.product_types_config[product_type_id]
608
+ config["_id"] = product_type_id
609
+ if "alias" in config:
610
+ product_type_id = config["alias"]
611
+ product_type = {"ID": product_type_id, **config}
612
+ if product_type not in product_types:
613
+ product_types.append(product_type)
614
+
563
615
  # Return the product_types sorted in lexicographic order of their ID
564
616
  return sorted(product_types, key=itemgetter("ID"))
565
617
 
566
618
  def fetch_product_types_list(self, provider: Optional[str] = None) -> None:
567
619
  """Fetch product types list and update if needed
568
620
 
569
- :param provider: (optional) The name of a provider for which product types list
570
- should be updated. Defaults to all providers (None value).
571
- :type provider: str
621
+ :param provider: The name of a provider or provider-group for which product types
622
+ list should be updated. Defaults to all providers (None value).
572
623
  """
624
+ providers_to_fetch = list(self.providers_config.keys())
625
+ # check if some providers are grouped under a group name which is not a provider name
573
626
  if provider is not None and provider not in self.providers_config:
574
- return
627
+ providers_to_fetch = [
628
+ p
629
+ for p, pconf in self.providers_config.items()
630
+ if provider == getattr(pconf, "group", None)
631
+ ]
632
+ if providers_to_fetch:
633
+ logger.info(
634
+ f"Fetch product types for {provider} group: {', '.join(providers_to_fetch)}"
635
+ )
636
+ else:
637
+ return None
638
+ elif provider is not None:
639
+ providers_to_fetch = [provider]
575
640
 
576
641
  # providers discovery confs that are fetchable
577
642
  providers_discovery_configs_fetchable: Dict[str, Any] = {}
578
643
  # check if any provider has not already been fetched for product types
579
644
  already_fetched = True
580
- for provider_to_fetch, provider_config in (
581
- {provider: self.providers_config[provider]}.items()
582
- if provider
583
- else self.providers_config.items()
584
- ):
645
+ for provider_to_fetch in providers_to_fetch:
646
+ provider_config = self.providers_config[provider_to_fetch]
585
647
  # get discovery conf
586
648
  if hasattr(provider_config, "search"):
587
649
  provider_search_config = provider_config.search
@@ -611,9 +673,8 @@ class EODataAccessGateway:
611
673
 
612
674
  if not ext_product_types_conf:
613
675
  # empty ext_product_types conf
614
- discover_kwargs = dict(provider=provider) if provider else {}
615
- ext_product_types_conf = self.discover_product_types(
616
- **discover_kwargs
676
+ ext_product_types_conf = (
677
+ self.discover_product_types(provider=provider) or {}
617
678
  )
618
679
 
619
680
  # update eodag product types list with new conf
@@ -691,36 +752,50 @@ class EODataAccessGateway:
691
752
  # providers not skipped here should be user-modified
692
753
  # or not in ext_product_types_conf (if eodag system conf != eodag conf used for ext_product_types_conf)
693
754
 
694
- # discover product types for user configured provider
695
- provider_ext_product_types_conf = self.discover_product_types(
696
- provider=provider
697
- )
698
-
699
- # update eodag product types list with new conf
700
- self.update_product_types_list(provider_ext_product_types_conf)
755
+ if not already_fetched:
756
+ # discover product types for user configured provider
757
+ provider_ext_product_types_conf = (
758
+ self.discover_product_types(provider=provider) or {}
759
+ )
760
+ # update eodag product types list with new conf
761
+ self.update_product_types_list(provider_ext_product_types_conf)
701
762
 
702
763
  def discover_product_types(
703
764
  self, provider: Optional[str] = None
704
765
  ) -> Optional[Dict[str, Any]]:
705
766
  """Fetch providers for product types
706
767
 
707
- :param provider: (optional) The name of a provider to fetch. Defaults to all
708
- providers (None value).
709
- :type provider: str
768
+ :param provider: The name of a provider or provider-group to fetch. Defaults to
769
+ all providers (None value).
710
770
  :returns: external product types configuration
711
- :rtype: dict
712
771
  """
772
+ grouped_providers = [
773
+ p
774
+ for p, provider_config in self.providers_config.items()
775
+ if provider == getattr(provider_config, "group", None)
776
+ ]
777
+ if provider and provider not in self.providers_config and grouped_providers:
778
+ logger.info(
779
+ f"Discover product types for {provider} group: {', '.join(grouped_providers)}"
780
+ )
781
+ elif provider and provider not in self.providers_config:
782
+ raise UnsupportedProvider(
783
+ f"The requested provider is not (yet) supported: {provider}"
784
+ )
713
785
  ext_product_types_conf: Dict[str, Any] = {}
714
786
  providers_to_fetch = [
715
787
  p
716
788
  for p in (
717
789
  [
718
- provider,
790
+ p
791
+ for p in self.providers_config
792
+ if p in grouped_providers + [provider]
719
793
  ]
720
794
  if provider
721
795
  else self.available_providers()
722
796
  )
723
797
  ]
798
+ kwargs: Dict[str, Any] = {}
724
799
  for provider in providers_to_fetch:
725
800
  if hasattr(self.providers_config[provider], "search"):
726
801
  search_plugin_config = self.providers_config[provider].search
@@ -728,36 +803,37 @@ class EODataAccessGateway:
728
803
  search_plugin_config = self.providers_config[provider].api
729
804
  else:
730
805
  return None
731
- if getattr(search_plugin_config, "discover_product_types", None):
806
+ if getattr(search_plugin_config, "discover_product_types", {}).get(
807
+ "fetch_url", None
808
+ ):
732
809
  search_plugin: Union[Search, Api] = next(
733
810
  self._plugins_manager.get_search_plugins(provider=provider)
734
811
  )
812
+ # check after plugin init if still fetchable
813
+ if not getattr(search_plugin.config, "discover_product_types", {}).get(
814
+ "fetch_url"
815
+ ):
816
+ continue
735
817
  # append auth to search plugin if needed
736
818
  if getattr(search_plugin.config, "need_auth", False):
737
- auth_plugin = self._plugins_manager.get_auth_plugin(
738
- search_plugin.provider
739
- )
740
- if callable(getattr(auth_plugin, "authenticate", None)):
741
- try:
742
- search_plugin.auth = auth_plugin.authenticate()
743
- except (AuthenticationError, MisconfiguredError) as e:
744
- logger.warning(
745
- f"Could not authenticate on {provider}: {str(e)}"
746
- )
747
- ext_product_types_conf[provider] = None
748
- continue
819
+ if auth := self._plugins_manager.get_auth(
820
+ search_plugin.provider,
821
+ getattr(search_plugin.config, "api_endpoint", None),
822
+ search_plugin.config,
823
+ ):
824
+ kwargs["auth"] = auth
749
825
  else:
750
- logger.warning(
751
- f"Could not authenticate on {provider} using {auth_plugin} plugin"
826
+ logger.debug(
827
+ f"Could not authenticate on {provider} for product types discovery"
752
828
  )
753
829
  ext_product_types_conf[provider] = None
754
830
  continue
755
831
 
756
- ext_product_types_conf[
757
- provider
758
- ] = search_plugin.discover_product_types()
832
+ ext_product_types_conf[provider] = search_plugin.discover_product_types(
833
+ **kwargs
834
+ )
759
835
 
760
- return ext_product_types_conf
836
+ return sort_dict(ext_product_types_conf)
761
837
 
762
838
  def update_product_types_list(
763
839
  self, ext_product_types_conf: Dict[str, Optional[Dict[str, Dict[str, Any]]]]
@@ -765,7 +841,6 @@ class EODataAccessGateway:
765
841
  """Update eodag product types list
766
842
 
767
843
  :param ext_product_types_conf: external product types configuration
768
- :type ext_product_types_conf: dict
769
844
  """
770
845
  for provider, new_product_types_conf in ext_product_types_conf.items():
771
846
  if new_product_types_conf and provider in self.providers_config:
@@ -775,7 +850,9 @@ class EODataAccessGateway:
775
850
  ) or getattr(self.providers_config[provider], "api", None)
776
851
  if search_plugin_config is None:
777
852
  continue
778
- if not hasattr(search_plugin_config, "discover_product_types"):
853
+ if not getattr(
854
+ search_plugin_config, "discover_product_types", {}
855
+ ).get("fetch_url", None):
779
856
  # conf has been updated and provider product types are no more discoverable
780
857
  continue
781
858
  provider_products_config = (
@@ -848,32 +925,53 @@ class EODataAccessGateway:
848
925
  # rebuild index after product types list update
849
926
  self.build_index()
850
927
 
851
- def available_providers(self, product_type: Optional[str] = None) -> List[str]:
852
- """Gives the sorted list of the available providers
928
+ def available_providers(
929
+ self, product_type: Optional[str] = None, by_group: bool = False
930
+ ) -> List[str]:
931
+ """Gives the sorted list of the available providers or groups
932
+
933
+ The providers or groups are sorted first by their priority level in descending order,
934
+ and then alphabetically in ascending order for providers or groups with the same
935
+ priority level.
853
936
 
854
937
  :param product_type: (optional) Only list providers configured for this product_type
855
- :type product_type: str
856
- :returns: the sorted list of the available providers
857
- :rtype: list
938
+ :param by_group: (optional) If set to True, list groups when available instead
939
+ of providers, mixed with other providers
940
+ :returns: the sorted list of the available providers or groups
858
941
  """
859
942
 
860
943
  if product_type:
861
- return sorted(
862
- k
944
+ providers = [
945
+ (v.group if by_group and hasattr(v, "group") else k, v.priority)
863
946
  for k, v in self.providers_config.items()
864
947
  if product_type in getattr(v, "products", {}).keys()
865
- )
948
+ ]
866
949
  else:
867
- return sorted(tuple(self.providers_config.keys()))
950
+ providers = [
951
+ (v.group if by_group and hasattr(v, "group") else k, v.priority)
952
+ for k, v in self.providers_config.items()
953
+ ]
954
+
955
+ # If by_group is True, keep only the highest priority for each group
956
+ if by_group:
957
+ group_priority: Dict[str, int] = {}
958
+ for name, priority in providers:
959
+ if name not in group_priority or priority > group_priority[name]:
960
+ group_priority[name] = priority
961
+ providers = list(group_priority.items())
962
+
963
+ # Sort by priority (descending) and then by name (ascending)
964
+ providers.sort(key=lambda x: (-x[1], x[0]))
965
+
966
+ # Return only the names of the providers or groups
967
+ return [name for name, _ in providers]
868
968
 
869
969
  def get_product_type_from_alias(self, alias_or_id: str) -> str:
870
970
  """Return the ID of a product type by either its ID or alias
871
971
 
872
972
  :param alias_or_id: Alias of the product type. If an existing ID is given, this
873
973
  method will directly return the given value.
874
- :type alias_or_id: str
875
974
  :returns: Internal name of the product type.
876
- :rtype: str
877
975
  """
878
976
  product_types = [
879
977
  k
@@ -901,56 +999,118 @@ class EODataAccessGateway:
901
999
  given product type, its ID is returned instead.
902
1000
 
903
1001
  :param product_type: product type ID
904
- :type product_type: str
905
1002
  :returns: Alias of the product type or its ID if no alias has been defined for it.
906
- :rtype: str
907
1003
  """
908
1004
  if product_type not in self.product_types_config:
909
1005
  raise NoMatchingProductType(product_type)
910
1006
 
911
1007
  return self.product_types_config[product_type].get("alias", product_type)
912
1008
 
913
- def guess_product_type(self, **kwargs: Any) -> List[str]:
914
- """Find eodag product types codes that best match a set of search params
915
-
916
- :param kwargs: A set of search parameters as keywords arguments
917
- :returns: The best match for the given parameters
918
- :rtype: list[str]
1009
+ def guess_product_type(
1010
+ self,
1011
+ free_text: Optional[str] = None,
1012
+ intersect: bool = False,
1013
+ instrument: Optional[str] = None,
1014
+ platform: Optional[str] = None,
1015
+ platformSerialIdentifier: Optional[str] = None,
1016
+ processingLevel: Optional[str] = None,
1017
+ sensorType: Optional[str] = None,
1018
+ keywords: Optional[str] = None,
1019
+ abstract: Optional[str] = None,
1020
+ title: Optional[str] = None,
1021
+ missionStartDate: Optional[str] = None,
1022
+ missionEndDate: Optional[str] = None,
1023
+ **kwargs: Any,
1024
+ ) -> List[str]:
1025
+ """
1026
+ Find EODAG product type IDs that best match a set of search parameters.
1027
+
1028
+ See https://whoosh.readthedocs.io/en/latest/querylang.html#the-default-query-language
1029
+ for syntax.
1030
+
1031
+ :param free_text: Whoosh-compatible free text search filter used to search
1032
+ accross all the following parameters
1033
+ :param intersect: Join results for each parameter using INTERSECT instead of UNION.
1034
+ :param instrument: Instrument parameter.
1035
+ :param platform: Platform parameter.
1036
+ :param platformSerialIdentifier: Platform serial identifier parameter.
1037
+ :param processingLevel: Processing level parameter.
1038
+ :param sensorType: Sensor type parameter.
1039
+ :param keywords: Keywords parameter.
1040
+ :param abstract: Abstract parameter.
1041
+ :param title: Title parameter.
1042
+ :param missionStartDate: start date for datetime filtering. Not used by free_text
1043
+ :param missionEndDate: end date for datetime filtering. Not used by free_text
1044
+ :returns: The best match for the given parameters.
919
1045
  :raises: :class:`~eodag.utils.exceptions.NoMatchingProductType`
920
1046
  """
921
- if kwargs.get("productType", None):
922
- return [kwargs["productType"]]
923
- supported_params = {
924
- param
925
- for param in (
926
- "instrument",
927
- "platform",
928
- "platformSerialIdentifier",
929
- "processingLevel",
930
- "sensorType",
931
- "keywords",
932
- "md5",
933
- )
934
- if kwargs.get(param, None) is not None
1047
+ if productType := kwargs.get("productType"):
1048
+ return [productType]
1049
+
1050
+ if not self._product_types_index:
1051
+ raise EodagError("Missing product types index")
1052
+
1053
+ filters = {
1054
+ "instrument": instrument,
1055
+ "platform": platform,
1056
+ "platformSerialIdentifier": platformSerialIdentifier,
1057
+ "processingLevel": processingLevel,
1058
+ "sensorType": sensorType,
1059
+ "keywords": keywords,
1060
+ "abstract": abstract,
1061
+ "title": title,
935
1062
  }
1063
+ joint = " AND " if intersect else " OR "
1064
+ filters_text = joint.join(
1065
+ [f"{k}:({v})" for k, v in filters.items() if v is not None]
1066
+ )
1067
+
1068
+ text = f"({free_text})" if free_text else ""
1069
+ if free_text and filters_text:
1070
+ text += joint
1071
+ if filters_text:
1072
+ text += f"({filters_text})"
1073
+
1074
+ if not text and (missionStartDate or missionEndDate):
1075
+ text = "*"
1076
+
936
1077
  with self._product_types_index.searcher() as searcher:
937
- results = None
938
- # For each search key, do a guess and then upgrade the result (i.e. when
939
- # merging results, if a hit appears in both results, its position is raised
940
- # to the top. This way, the top most result will be the hit that best
941
- # matches the given queries. Put another way, this best guess is the one
942
- # that crosses the highest number of search params from the given queries
943
- for search_key in supported_params:
944
- query = QueryParser(search_key, self._product_types_index.schema).parse(
945
- kwargs[search_key]
1078
+ p = EODAGQueryParser(list(filters.keys()), self._product_types_index.schema)
1079
+ query = p.parse(text)
1080
+ results = searcher.search(query, limit=None)
1081
+
1082
+ guesses: List[Dict[str, str]] = [dict(r) for r in results or []]
1083
+
1084
+ # datetime filtering
1085
+ if missionStartDate or missionEndDate:
1086
+ min_aware = datetime.datetime.min.replace(tzinfo=datetime.timezone.utc)
1087
+ max_aware = datetime.datetime.max.replace(tzinfo=datetime.timezone.utc)
1088
+ guesses = [
1089
+ g
1090
+ for g in guesses
1091
+ if (
1092
+ max(
1093
+ rfc3339_str_to_datetime(missionStartDate)
1094
+ if missionStartDate
1095
+ else min_aware,
1096
+ rfc3339_str_to_datetime(g["missionStartDate"])
1097
+ if g.get("missionStartDate")
1098
+ else min_aware,
1099
+ )
1100
+ <= min(
1101
+ rfc3339_str_to_datetime(missionEndDate)
1102
+ if missionEndDate
1103
+ else max_aware,
1104
+ rfc3339_str_to_datetime(g["missionEndDate"])
1105
+ if g.get("missionEndDate")
1106
+ else max_aware,
1107
+ )
946
1108
  )
947
- if results is None:
948
- results = searcher.search(query, limit=None)
949
- else:
950
- results.upgrade_and_extend(searcher.search(query, limit=None))
951
- guesses: List[str] = [r["ID"] for r in results or []]
1109
+ ]
1110
+
952
1111
  if guesses:
953
- return guesses
1112
+ return [g["ID"] for g in guesses or []]
1113
+
954
1114
  raise NoMatchingProductType()
955
1115
 
956
1116
  def search(
@@ -963,8 +1123,9 @@ class EODataAccessGateway:
963
1123
  geom: Optional[Union[str, Dict[str, float], BaseGeometry]] = None,
964
1124
  locations: Optional[Dict[str, str]] = None,
965
1125
  provider: Optional[str] = None,
1126
+ count: bool = False,
966
1127
  **kwargs: Any,
967
- ) -> Tuple[SearchResult, int]:
1128
+ ) -> SearchResult:
968
1129
  """Look for products matching criteria on known providers.
969
1130
 
970
1131
  The default behaviour is to look for products on the provider with the
@@ -975,21 +1136,16 @@ class EODataAccessGateway:
975
1136
  Only if the request fails for all available providers, an error will be thrown.
976
1137
 
977
1138
  :param page: (optional) The page number to return
978
- :type page: int
979
1139
  :param items_per_page: (optional) The number of results that must appear in one single
980
1140
  page
981
- :type items_per_page: int
982
1141
  :param raise_errors: (optional) When an error occurs when searching, if this is set to
983
1142
  True, the error is raised
984
- :type raise_errors: bool
985
1143
  :param start: (optional) Start sensing time in ISO 8601 format (e.g. "1990-11-26",
986
1144
  "1990-11-26T14:30:10.153Z", "1990-11-26T14:30:10+02:00", ...).
987
1145
  If no time offset is given, the time is assumed to be given in UTC.
988
- :type start: str
989
1146
  :param end: (optional) End sensing time in ISO 8601 format (e.g. "1990-11-26",
990
1147
  "1990-11-26T14:30:10.153Z", "1990-11-26T14:30:10+02:00", ...).
991
1148
  If no time offset is given, the time is assumed to be given in UTC.
992
- :type end: str
993
1149
  :param geom: (optional) Search area that can be defined in different ways:
994
1150
 
995
1151
  * with a Shapely geometry object:
@@ -999,23 +1155,22 @@ class EODataAccessGateway:
999
1155
  * with a bounding box as list of float:
1000
1156
  ``[lonmin, latmin, lonmax, latmax]``
1001
1157
  * with a WKT str
1002
- :type geom: Union[str, dict, shapely.geometry.base.BaseGeometry]
1003
1158
  :param locations: (optional) Location filtering by name using locations configuration
1004
1159
  ``{"<location_name>"="<attr_regex>"}``. For example, ``{"country"="PA."}`` will use
1005
1160
  the geometry of the features having the property ISO3 starting with
1006
1161
  'PA' such as Panama and Pakistan in the shapefile configured with
1007
1162
  name=country and attr=ISO3
1008
- :type locations: dict
1009
- :param kwargs: Some other criteria that will be used to do the search,
1010
- using paramaters compatibles with the provider
1011
1163
  :param provider: (optional) the provider to be used. If set, search fallback will be disabled.
1012
1164
  If not set, the configured preferred provider will be used at first
1013
1165
  before trying others until finding results.
1014
- :type provider: str
1015
- :type kwargs: Union[int, str, bool, dict]
1016
- :returns: A collection of EO products matching the criteria and the total
1017
- number of results found
1018
- :rtype: tuple(:class:`~eodag.api.search_result.SearchResult`, int)
1166
+ :param count: (optional) Whether to run a query with a count request or not
1167
+ :param kwargs: Some other criteria that will be used to do the search,
1168
+ using paramaters compatibles with the provider
1169
+ :returns: A collection of EO products matching the criteria
1170
+
1171
+ .. versionchanged:: v3.0.0b1
1172
+ ``search()`` method now returns only a single :class:`~eodag.api.search_result.SearchResult`
1173
+ instead of a 2 values tuple.
1019
1174
 
1020
1175
  .. note::
1021
1176
  The search interfaces, which are implemented as plugins, are required to
@@ -1030,16 +1185,12 @@ class EODataAccessGateway:
1030
1185
  provider=provider,
1031
1186
  **kwargs,
1032
1187
  )
1033
-
1034
1188
  if search_kwargs.get("id"):
1035
- # adds minimal pagination to be able to check only 1 product is returned
1036
- search_kwargs.update(
1037
- page=1,
1038
- items_per_page=2,
1039
- raise_errors=raise_errors,
1040
- )
1041
1189
  return self._search_by_id(
1042
- search_kwargs.pop("id"), provider=provider, **search_kwargs
1190
+ search_kwargs.pop("id"),
1191
+ provider=provider,
1192
+ raise_errors=raise_errors,
1193
+ **search_kwargs,
1043
1194
  )
1044
1195
  # remove datacube query string from kwargs which was only needed for search-by-id
1045
1196
  search_kwargs.pop("_dc_qs", None)
@@ -1049,26 +1200,29 @@ class EODataAccessGateway:
1049
1200
  items_per_page=items_per_page,
1050
1201
  )
1051
1202
 
1052
- self.search_errors = set()
1203
+ errors: List[Tuple[str, Exception]] = []
1053
1204
  # Loop over available providers and return the first non-empty results
1054
1205
  for i, search_plugin in enumerate(search_plugins):
1055
1206
  search_plugin.clear()
1056
- search_results, total_results = self._do_search(
1207
+ search_results = self._do_search(
1057
1208
  search_plugin,
1058
- count=True,
1209
+ count=count,
1059
1210
  raise_errors=raise_errors,
1060
1211
  **search_kwargs,
1061
1212
  )
1213
+ errors.extend(search_results.errors)
1062
1214
  if len(search_results) == 0 and i < len(search_plugins) - 1:
1063
1215
  logger.warning(
1064
1216
  f"No result could be obtained from provider {search_plugin.provider}, "
1065
1217
  "we will try to get the data from another provider",
1066
1218
  )
1067
1219
  elif len(search_results) > 0:
1068
- return search_results, total_results
1220
+ search_results.errors = errors
1221
+ return search_results
1069
1222
 
1070
- logger.error("No result could be obtained from any available provider")
1071
- return SearchResult([]), 0
1223
+ if i > 1:
1224
+ logger.error("No result could be obtained from any available provider")
1225
+ return SearchResult([], 0, errors) if count else SearchResult([], errors=errors)
1072
1226
 
1073
1227
  def search_iter_page(
1074
1228
  self,
@@ -1082,15 +1236,12 @@ class EODataAccessGateway:
1082
1236
  """Iterate over the pages of a products search.
1083
1237
 
1084
1238
  :param items_per_page: (optional) The number of results requested per page
1085
- :type items_per_page: int
1086
1239
  :param start: (optional) Start sensing time in ISO 8601 format (e.g. "1990-11-26",
1087
1240
  "1990-11-26T14:30:10.153Z", "1990-11-26T14:30:10+02:00", ...).
1088
1241
  If no time offset is given, the time is assumed to be given in UTC.
1089
- :type start: str
1090
1242
  :param end: (optional) End sensing time in ISO 8601 format (e.g. "1990-11-26",
1091
1243
  "1990-11-26T14:30:10.153Z", "1990-11-26T14:30:10+02:00", ...).
1092
1244
  If no time offset is given, the time is assumed to be given in UTC.
1093
- :type end: str
1094
1245
  :param geom: (optional) Search area that can be defined in different ways:
1095
1246
 
1096
1247
  * with a Shapely geometry object:
@@ -1100,19 +1251,15 @@ class EODataAccessGateway:
1100
1251
  * with a bounding box as list of float:
1101
1252
  ``[lonmin, latmin, lonmax, latmax]``
1102
1253
  * with a WKT str
1103
- :type geom: Union[str, dict, shapely.geometry.base.BaseGeometry]
1104
1254
  :param locations: (optional) Location filtering by name using locations configuration
1105
1255
  ``{"<location_name>"="<attr_regex>"}``. For example, ``{"country"="PA."}`` will use
1106
1256
  the geometry of the features having the property ISO3 starting with
1107
1257
  'PA' such as Panama and Pakistan in the shapefile configured with
1108
1258
  name=country and attr=ISO3
1109
- :type locations: dict
1110
1259
  :param kwargs: Some other criteria that will be used to do the search,
1111
1260
  using paramaters compatibles with the provider
1112
- :type kwargs: Union[int, str, bool, dict]
1113
1261
  :returns: An iterator that yields page per page a collection of EO products
1114
1262
  matching the criteria
1115
- :rtype: Iterator[:class:`~eodag.api.search_result.SearchResult`]
1116
1263
  """
1117
1264
  search_plugins, search_kwargs = self._prepare_search(
1118
1265
  start=start, end=end, geom=geom, locations=locations, **kwargs
@@ -1147,15 +1294,11 @@ class EODataAccessGateway:
1147
1294
  """Iterate over the pages of a products search using a given search plugin.
1148
1295
 
1149
1296
  :param items_per_page: (optional) The number of results requested per page
1150
- :type items_per_page: int
1151
1297
  :param kwargs: Some other criteria that will be used to do the search,
1152
1298
  using parameters compatibles with the provider
1153
- :type kwargs: Union[int, str, bool, dict]
1154
1299
  :param search_plugin: search plugin to be used
1155
- :type search_plugin: eodag.plugins.search.base.Search
1156
1300
  :returns: An iterator that yields page per page a collection of EO products
1157
1301
  matching the criteria
1158
- :rtype: Iterator[:class:`~eodag.api.search_result.SearchResult`]
1159
1302
  """
1160
1303
 
1161
1304
  iteration = 1
@@ -1181,9 +1324,10 @@ class EODataAccessGateway:
1181
1324
  pagination_config["next_page_query_obj"] = next_page_query_obj
1182
1325
  logger.info("Iterate search over multiple pages: page #%s", iteration)
1183
1326
  try:
1184
- if "raise_errors" in kwargs:
1185
- kwargs.pop("raise_errors")
1186
- products, _ = self._do_search(
1327
+ # remove unwanted kwargs for _do_search
1328
+ kwargs.pop("count", None)
1329
+ kwargs.pop("raise_errors", None)
1330
+ search_result = self._do_search(
1187
1331
  search_plugin, count=False, raise_errors=True, **kwargs
1188
1332
  )
1189
1333
  except Exception:
@@ -1222,12 +1366,12 @@ class EODataAccessGateway:
1222
1366
  else:
1223
1367
  search_plugin.next_page_query_obj = next_page_query_obj
1224
1368
 
1225
- if len(products) > 0:
1369
+ if len(search_result) > 0:
1226
1370
  # The first products between two iterations are compared. If they
1227
1371
  # are actually the same product, it means the iteration failed at
1228
1372
  # progressing for some reason. This is implemented as a workaround
1229
1373
  # to some search plugins/providers not handling pagination.
1230
- product = products[0]
1374
+ product = search_result[0]
1231
1375
  if (
1232
1376
  prev_product
1233
1377
  and product.properties["id"] == prev_product.properties["id"]
@@ -1240,11 +1384,11 @@ class EODataAccessGateway:
1240
1384
  )
1241
1385
  last_page_with_products = iteration - 1
1242
1386
  break
1243
- yield products
1387
+ yield search_result
1244
1388
  prev_product = product
1245
1389
  # Prevent a last search if the current one returned less than the
1246
1390
  # maximum number of items asked for.
1247
- if len(products) < items_per_page:
1391
+ if len(search_result) < items_per_page:
1248
1392
  last_page_with_products = iteration
1249
1393
  break
1250
1394
  else:
@@ -1283,15 +1427,12 @@ class EODataAccessGateway:
1283
1427
  matching the search criteria. If this number is not
1284
1428
  available, a default value of 50 is used instead.
1285
1429
  items_per_page can also be set to any arbitrary value.
1286
- :type items_per_page: int
1287
1430
  :param start: (optional) Start sensing time in ISO 8601 format (e.g. "1990-11-26",
1288
1431
  "1990-11-26T14:30:10.153Z", "1990-11-26T14:30:10+02:00", ...).
1289
1432
  If no time offset is given, the time is assumed to be given in UTC.
1290
- :type start: str
1291
1433
  :param end: (optional) End sensing time in ISO 8601 format (e.g. "1990-11-26",
1292
1434
  "1990-11-26T14:30:10.153Z", "1990-11-26T14:30:10+02:00", ...).
1293
1435
  If no time offset is given, the time is assumed to be given in UTC.
1294
- :type end: str
1295
1436
  :param geom: (optional) Search area that can be defined in different ways:
1296
1437
 
1297
1438
  * with a Shapely geometry object:
@@ -1301,25 +1442,21 @@ class EODataAccessGateway:
1301
1442
  * with a bounding box as list of float:
1302
1443
  ``[lonmin, latmin, lonmax, latmax]``
1303
1444
  * with a WKT str
1304
- :type geom: Union[str, dict, shapely.geometry.base.BaseGeometry]
1305
1445
  :param locations: (optional) Location filtering by name using locations configuration
1306
1446
  ``{"<location_name>"="<attr_regex>"}``. For example, ``{"country"="PA."}`` will use
1307
1447
  the geometry of the features having the property ISO3 starting with
1308
1448
  'PA' such as Panama and Pakistan in the shapefile configured with
1309
1449
  name=country and attr=ISO3
1310
- :type locations: dict
1311
1450
  :param kwargs: Some other criteria that will be used to do the search,
1312
1451
  using parameters compatible with the provider
1313
- :type kwargs: Union[int, str, bool, dict]
1314
1452
  :returns: An iterator that yields page per page a collection of EO products
1315
1453
  matching the criteria
1316
- :rtype: Iterator[:class:`~eodag.api.search_result.SearchResult`]
1317
1454
  """
1318
1455
  # Get the search plugin and the maximized value
1319
1456
  # of items_per_page if defined for the provider used.
1320
1457
  try:
1321
1458
  product_type = self.get_product_type_from_alias(
1322
- kwargs.get("productType", None) or self.guess_product_type(**kwargs)[0]
1459
+ self.guess_product_type(**kwargs)[0]
1323
1460
  )
1324
1461
  except NoMatchingProductType:
1325
1462
  product_type = GENERIC_PRODUCT_TYPE
@@ -1338,10 +1475,14 @@ class EODataAccessGateway:
1338
1475
  start=start, end=end, geom=geom, locations=locations, **kwargs
1339
1476
  )
1340
1477
  for i, search_plugin in enumerate(search_plugins):
1341
- itp = items_per_page or search_plugin.config.pagination.get(
1342
- "max_items_per_page", DEFAULT_MAX_ITEMS_PER_PAGE
1478
+ itp = (
1479
+ items_per_page
1480
+ or getattr(search_plugin.config, "pagination", {}).get(
1481
+ "max_items_per_page"
1482
+ )
1483
+ or DEFAULT_MAX_ITEMS_PER_PAGE
1343
1484
  )
1344
- logger.debug(
1485
+ logger.info(
1345
1486
  "Searching for all the products with provider %s and a maximum of %s "
1346
1487
  "items per page.",
1347
1488
  search_plugin.provider,
@@ -1385,7 +1526,7 @@ class EODataAccessGateway:
1385
1526
 
1386
1527
  def _search_by_id(
1387
1528
  self, uid: str, provider: Optional[str] = None, **kwargs: Any
1388
- ) -> Tuple[SearchResult, int]:
1529
+ ) -> SearchResult:
1389
1530
  """Internal method that enables searching a product by its id.
1390
1531
 
1391
1532
  Keeps requesting providers until a result matching the id is supplied. The
@@ -1398,16 +1539,11 @@ class EODataAccessGateway:
1398
1539
  perform the search, if this information is available
1399
1540
 
1400
1541
  :param uid: The uid of the EO product
1401
- :type uid: str
1402
1542
  :param provider: (optional) The provider on which to search the product.
1403
1543
  This may be useful for performance reasons when the user
1404
1544
  knows this product is available on the given provider
1405
- :type provider: str
1406
1545
  :param kwargs: Search criteria to help finding the right product
1407
- :type kwargs: Any
1408
- :returns: A search result with one EO product or None at all, and the number
1409
- of EO products retrieved (0 or 1)
1410
- :rtype: tuple(:class:`~eodag.api.search_result.SearchResult`, int)
1546
+ :returns: A search result with one EO product or None at all
1411
1547
  """
1412
1548
  product_type = kwargs.get("productType", None)
1413
1549
  if product_type is not None:
@@ -1422,16 +1558,52 @@ class EODataAccessGateway:
1422
1558
  # datacube query string
1423
1559
  _dc_qs = kwargs.pop("_dc_qs", None)
1424
1560
 
1561
+ results = SearchResult([])
1562
+
1425
1563
  for plugin in search_plugins:
1426
1564
  logger.info(
1427
1565
  "Searching product with id '%s' on provider: %s", uid, plugin.provider
1428
1566
  )
1429
1567
  logger.debug("Using plugin class for search: %s", plugin.__class__.__name__)
1430
1568
  plugin.clear()
1431
- if isinstance(plugin, BuildPostSearchResult):
1432
- results, _ = self._do_search(plugin, id=uid, _dc_qs=_dc_qs, **kwargs)
1569
+
1570
+ # adds maximal pagination to be able to do a search-all + crunch if more
1571
+ # than one result are returned
1572
+ items_per_page = plugin.config.pagination.get(
1573
+ "max_items_per_page", DEFAULT_MAX_ITEMS_PER_PAGE
1574
+ )
1575
+ kwargs.update(items_per_page=items_per_page)
1576
+ if isinstance(plugin, PostJsonSearch):
1577
+ kwargs.update(
1578
+ items_per_page=items_per_page,
1579
+ _dc_qs=_dc_qs,
1580
+ )
1433
1581
  else:
1434
- results, _ = self._do_search(plugin, id=uid, **kwargs)
1582
+ kwargs.update(
1583
+ items_per_page=items_per_page,
1584
+ )
1585
+
1586
+ try:
1587
+ # if more than one results are found, try getting them all and then filter using crunch
1588
+ for page_results in self.search_iter_page_plugin(
1589
+ search_plugin=plugin,
1590
+ id=uid,
1591
+ **kwargs,
1592
+ ):
1593
+ results.data.extend(page_results.data)
1594
+ except Exception as e:
1595
+ if kwargs.get("raise_errors"):
1596
+ raise
1597
+ logger.warning(e)
1598
+ continue
1599
+
1600
+ # try using crunch to get unique result
1601
+ if (
1602
+ len(results) > 1
1603
+ and len(filtered := results.filter_property(id=uid)) == 1
1604
+ ):
1605
+ results = filtered
1606
+
1435
1607
  if len(results) == 1:
1436
1608
  if not results[0].product_type:
1437
1609
  # guess product type from properties
@@ -1439,20 +1611,36 @@ class EODataAccessGateway:
1439
1611
  results[0].product_type = guesses[0]
1440
1612
  # reset driver
1441
1613
  results[0].driver = results[0].get_driver()
1442
- return results, 1
1614
+ results.number_matched = 1
1615
+ return results
1443
1616
  elif len(results) > 1:
1444
- if getattr(plugin.config, "two_passes_id_search", False):
1445
- # check if id of one product exactly matches id that was searched for
1446
- # required if provider does not offer search by id and therefore other
1447
- # parameters which might not given an exact result are used
1448
- for result in results:
1449
- if result.properties["id"] == uid.split(".")[0]:
1450
- return [results[0]], 1
1451
1617
  logger.info(
1452
1618
  "Several products found for this id (%s). You may try searching using more selective criteria.",
1453
1619
  results,
1454
1620
  )
1455
- return SearchResult([]), 0
1621
+ return SearchResult([], 0)
1622
+
1623
+ def _fetch_external_product_type(self, provider: str, product_type: str):
1624
+ plugins = self._plugins_manager.get_search_plugins(provider=provider)
1625
+ plugin = next(plugins)
1626
+
1627
+ # check after plugin init if still fetchable
1628
+ if not getattr(plugin.config, "discover_product_types", {}).get("fetch_url"):
1629
+ return None
1630
+
1631
+ kwargs: Dict[str, Any] = {"productType": product_type}
1632
+
1633
+ # append auth if needed
1634
+ if getattr(plugin.config, "need_auth", False):
1635
+ if auth := self._plugins_manager.get_auth(
1636
+ plugin.provider,
1637
+ getattr(plugin.config, "api_endpoint", None),
1638
+ plugin.config,
1639
+ ):
1640
+ kwargs["auth"] = auth
1641
+
1642
+ product_type_config = plugin.discover_product_types(**kwargs)
1643
+ self.update_product_types_list({provider: product_type_config})
1456
1644
 
1457
1645
  def _prepare_search(
1458
1646
  self,
@@ -1480,25 +1668,18 @@ class EODataAccessGateway:
1480
1668
  :param start: (optional) Start sensing time in ISO 8601 format (e.g. "1990-11-26",
1481
1669
  "1990-11-26T14:30:10.153Z", "1990-11-26T14:30:10+02:00", ...).
1482
1670
  If no time offset is given, the time is assumed to be given in UTC.
1483
- :type start: str
1484
1671
  :param end: (optional) End sensing time in ISO 8601 format (e.g. "1990-11-26",
1485
1672
  "1990-11-26T14:30:10.153Z", "1990-11-26T14:30:10+02:00", ...).
1486
1673
  If no time offset is given, the time is assumed to be given in UTC.
1487
- :type end: str
1488
1674
  :param geom: (optional) Search area that can be defined in different ways (see search)
1489
- :type geom: Union[str, dict, shapely.geometry.base.BaseGeometry]
1490
1675
  :param locations: (optional) Location filtering by name using locations configuration
1491
- :type locations: dict
1492
1676
  :param provider: provider to be used, if no provider is given or the product type
1493
1677
  is not available for the provider, the preferred provider is used
1494
- :type provider: str
1495
1678
  :param kwargs: Some other criteria
1496
1679
  * id and/or a provider for a search by
1497
1680
  * search criteria to guess the product type
1498
1681
  * other criteria compatible with the provider
1499
- :type kwargs: Any
1500
1682
  :returns: Search plugins list and the prepared kwargs to make a query.
1501
- :rtype: tuple(list, dict)
1502
1683
  """
1503
1684
  product_type = kwargs.get("productType", None)
1504
1685
  if product_type is None:
@@ -1532,7 +1713,7 @@ class EODataAccessGateway:
1532
1713
  try:
1533
1714
  product_type = self.get_product_type_from_alias(product_type)
1534
1715
  except NoMatchingProductType:
1535
- logger.warning("unknown product type " + product_type)
1716
+ logger.info("unknown product type " + product_type)
1536
1717
  kwargs["productType"] = product_type
1537
1718
 
1538
1719
  if start is not None:
@@ -1564,10 +1745,16 @@ class EODataAccessGateway:
1564
1745
  product_type
1565
1746
  not in self._plugins_manager.product_type_to_provider_config_map.keys()
1566
1747
  ):
1567
- logger.debug(
1568
- f"Fetching external product types sources to find {product_type} product type"
1569
- )
1570
- self.fetch_product_types_list()
1748
+ if provider:
1749
+ # Try to get specific product type from external provider
1750
+ logger.debug(f"Fetching {provider} to find {product_type} product type")
1751
+ self._fetch_external_product_type(provider, product_type)
1752
+ if not provider:
1753
+ # no provider or still not found -> fetch all external product types
1754
+ logger.debug(
1755
+ f"Fetching external product types sources to find {product_type} product type"
1756
+ )
1757
+ self.fetch_product_types_list()
1571
1758
 
1572
1759
  preferred_provider = self.get_preferred_provider()[0]
1573
1760
 
@@ -1589,12 +1776,10 @@ class EODataAccessGateway:
1589
1776
  provider = preferred_provider
1590
1777
  providers = [plugin.provider for plugin in search_plugins]
1591
1778
  if provider not in providers:
1592
- logger.warning(
1593
- "Product type '%s' is not available with provider '%s'. "
1594
- "Searching it on provider '%s' instead.",
1779
+ logger.debug(
1780
+ "Product type '%s' is not available with preferred provider '%s'.",
1595
1781
  product_type,
1596
1782
  provider,
1597
- search_plugins[0].provider,
1598
1783
  )
1599
1784
  else:
1600
1785
  provider_plugin = list(
@@ -1602,11 +1787,6 @@ class EODataAccessGateway:
1602
1787
  )[0]
1603
1788
  search_plugins.remove(provider_plugin)
1604
1789
  search_plugins.insert(0, provider_plugin)
1605
- logger.info(
1606
- "Searching product type '%s' on provider: %s",
1607
- product_type,
1608
- search_plugins[0].provider,
1609
- )
1610
1790
  # Add product_types_config to plugin config. This dict contains product
1611
1791
  # type metadata that will also be stored in each product's properties.
1612
1792
  for search_plugin in search_plugins:
@@ -1617,8 +1797,7 @@ class EODataAccessGateway:
1617
1797
  for p in self.list_product_types(
1618
1798
  search_plugin.provider, fetch_providers=False
1619
1799
  )
1620
- if p["ID"] == product_type
1621
- or ("_id" in p and p["_id"] == product_type)
1800
+ if p["_id"] == product_type
1622
1801
  ][0],
1623
1802
  **{"productType": product_type},
1624
1803
  )
@@ -1638,29 +1817,27 @@ class EODataAccessGateway:
1638
1817
  def _do_search(
1639
1818
  self,
1640
1819
  search_plugin: Union[Search, Api],
1641
- count: bool = True,
1820
+ count: bool = False,
1642
1821
  raise_errors: bool = False,
1643
1822
  **kwargs: Any,
1644
- ) -> Tuple[SearchResult, Optional[int]]:
1823
+ ) -> SearchResult:
1645
1824
  """Internal method that performs a search on a given provider.
1646
1825
 
1647
1826
  :param search_plugin: A search plugin
1648
- :type search_plugin: eodag.plugins.base.Search
1649
1827
  :param count: (optional) Whether to run a query with a count request or not
1650
- :type count: bool
1651
1828
  :param raise_errors: (optional) When an error occurs when searching, if this is set to
1652
1829
  True, the error is raised
1653
- :type raise_errors: bool
1654
1830
  :param kwargs: Some other criteria that will be used to do the search
1655
- :type kwargs: Any
1656
- :returns: A collection of EO products matching the criteria and the total
1657
- number of results found if count is True else None
1658
- :rtype: tuple(:class:`~eodag.api.search_result.SearchResult`, int or None)
1831
+ :returns: A collection of EO products matching the criteria
1659
1832
  """
1833
+ logger.info("Searching on provider %s", search_plugin.provider)
1660
1834
  max_items_per_page = getattr(search_plugin.config, "pagination", {}).get(
1661
1835
  "max_items_per_page", DEFAULT_MAX_ITEMS_PER_PAGE
1662
1836
  )
1663
- if kwargs.get("items_per_page", DEFAULT_ITEMS_PER_PAGE) > max_items_per_page:
1837
+ if (
1838
+ kwargs.get("items_per_page", DEFAULT_ITEMS_PER_PAGE) > max_items_per_page
1839
+ and max_items_per_page > 0
1840
+ ):
1664
1841
  logger.warning(
1665
1842
  "EODAG believes that you might have asked for more products/items "
1666
1843
  "than the maximum allowed by '%s': %s > %s. Try to lower "
@@ -1671,56 +1848,27 @@ class EODataAccessGateway:
1671
1848
  max_items_per_page,
1672
1849
  )
1673
1850
 
1674
- need_auth = getattr(search_plugin.config, "need_auth", False)
1675
- auth_plugin = self._plugins_manager.get_auth_plugin(search_plugin.provider)
1676
- can_authenticate = callable(getattr(auth_plugin, "authenticate", None))
1677
-
1678
1851
  results: List[EOProduct] = []
1679
- total_results = 0
1852
+ total_results: Optional[int] = 0 if count else None
1853
+
1854
+ errors: List[Tuple[str, Exception]] = []
1680
1855
 
1681
1856
  try:
1682
- if need_auth and auth_plugin and can_authenticate:
1683
- search_plugin.auth = auth_plugin.authenticate()
1684
-
1685
- res, nb_res = search_plugin.query(count=count, auth=auth_plugin, **kwargs)
1686
-
1687
- # Only do the pagination computations when it makes sense. For example,
1688
- # for a search by id, we can reasonably guess that the provider will return
1689
- # At most 1 product, so we don't need such a thing as pagination
1690
- page = kwargs.get("page")
1691
- items_per_page = kwargs.get("items_per_page")
1692
- if page and items_per_page and count:
1693
- # Take into account the fact that a provider may not return the count of
1694
- # products (in that case, fallback to using the length of the results it
1695
- # returned and the page requested. As an example, check the result of
1696
- # the following request (look for the value of properties.totalResults)
1697
- # https://theia-landsat.cnes.fr/resto/api/collections/Landsat/search.json?
1698
- # maxRecords=1&page=1
1699
- if not nb_res:
1700
- nb_res = len(res) * page
1701
-
1702
- # Attempt to ensure a little bit more coherence. Some providers return
1703
- # a fuzzy number of total results, meaning that you have to keep
1704
- # requesting it until it has returned everything it has to know exactly
1705
- # how many EO products they have in their stock. In that case, we need
1706
- # to replace the returned number of results with the sum of the number
1707
- # of items that were skipped so far and the length of the currently
1708
- # retrieved items. We know there is an incoherence when the number of
1709
- # skipped items is greater than the total number of items returned by
1710
- # the plugin
1711
- nb_skipped_items = items_per_page * (page - 1)
1712
- nb_current_items = len(res)
1713
- if nb_skipped_items > nb_res:
1714
- if nb_res != 0:
1715
- nb_res = nb_skipped_items + nb_current_items
1716
- # This is for when the returned results is an empty list and the
1717
- # number of results returned is incoherent with the observations.
1718
- # In that case, we assume the total number of results is the number
1719
- # of skipped results. By requesting a lower page than the current
1720
- # one, a user can iteratively reach the last page of results for
1721
- # these criteria on the provider.
1722
- else:
1723
- nb_res = nb_skipped_items
1857
+ prep = PreparedSearch(count=count)
1858
+
1859
+ # append auth if needed
1860
+ if getattr(search_plugin.config, "need_auth", False):
1861
+ if auth := self._plugins_manager.get_auth(
1862
+ search_plugin.provider,
1863
+ getattr(search_plugin.config, "api_endpoint", None),
1864
+ search_plugin.config,
1865
+ ):
1866
+ prep.auth = auth
1867
+
1868
+ prep.page = kwargs.pop("page", None)
1869
+ prep.items_per_page = kwargs.pop("items_per_page", None)
1870
+
1871
+ res, nb_res = search_plugin.query(prep, **kwargs)
1724
1872
 
1725
1873
  if not isinstance(res, list):
1726
1874
  raise PluginImplementationError(
@@ -1743,8 +1891,8 @@ class EODataAccessGateway:
1743
1891
  pattern = re.compile(r"[^\w,]+")
1744
1892
  try:
1745
1893
  guesses = self.guess_product_type(
1894
+ intersect=False,
1746
1895
  **{
1747
- # k:str(v) for k,v in eo_product.properties.items()
1748
1896
  k: pattern.sub("", str(v).upper())
1749
1897
  for k, v in eo_product.properties.items()
1750
1898
  if k
@@ -1757,7 +1905,7 @@ class EODataAccessGateway:
1757
1905
  "keywords",
1758
1906
  ]
1759
1907
  and v is not None
1760
- }
1908
+ },
1761
1909
  )
1762
1910
  except NoMatchingProductType:
1763
1911
  pass
@@ -1776,11 +1924,34 @@ class EODataAccessGateway:
1776
1924
  download_plugin = self._plugins_manager.get_download_plugin(
1777
1925
  eo_product
1778
1926
  )
1927
+ if len(eo_product.assets) > 0:
1928
+ matching_url = next(iter(eo_product.assets.values()))["href"]
1929
+ elif eo_product.properties.get("storageStatus") != ONLINE_STATUS:
1930
+ matching_url = eo_product.properties.get(
1931
+ "orderLink"
1932
+ ) or eo_product.properties.get("downloadLink")
1933
+ else:
1934
+ matching_url = eo_product.properties.get("downloadLink")
1935
+
1936
+ try:
1937
+ auth_plugin = next(
1938
+ self._plugins_manager.get_auth_plugins(
1939
+ search_plugin.provider,
1940
+ matching_url=matching_url,
1941
+ matching_conf=download_plugin.config,
1942
+ )
1943
+ )
1944
+ except StopIteration:
1945
+ auth_plugin = None
1779
1946
  eo_product.register_downloader(download_plugin, auth_plugin)
1780
1947
 
1781
1948
  results.extend(res)
1782
- total_results = None if nb_res is None else total_results + nb_res
1783
- if count:
1949
+ total_results = (
1950
+ None
1951
+ if (nb_res is None or total_results is None)
1952
+ else total_results + nb_res
1953
+ )
1954
+ if count and nb_res is not None:
1784
1955
  logger.info(
1785
1956
  "Found %s result(s) on provider '%s'",
1786
1957
  nb_res,
@@ -1801,10 +1972,6 @@ class EODataAccessGateway:
1801
1972
  "the total number of products matching the search criteria"
1802
1973
  )
1803
1974
  except Exception as e:
1804
- log_msg = f"No result from provider '{search_plugin.provider}' due to an error during search."
1805
- if not raise_errors:
1806
- log_msg += " Raise verbosity of log messages for details"
1807
- logger.info(log_msg)
1808
1975
  if raise_errors:
1809
1976
  # Raise the error, letting the application wrapping eodag know that
1810
1977
  # something went bad. This way it will be able to decide what to do next
@@ -1814,16 +1981,14 @@ class EODataAccessGateway:
1814
1981
  "Error while searching on provider %s (ignored):",
1815
1982
  search_plugin.provider,
1816
1983
  )
1817
- self.search_errors.add((search_plugin.provider, e))
1818
- return SearchResult(results), total_results
1984
+ errors.append((search_plugin.provider, e))
1985
+ return SearchResult(results, total_results, errors)
1819
1986
 
1820
1987
  def crunch(self, results: SearchResult, **kwargs: Any) -> SearchResult:
1821
1988
  """Apply the filters given through the keyword arguments to the results
1822
1989
 
1823
1990
  :param results: The results of a eodag search request
1824
- :type results: :class:`~eodag.api.search_result.SearchResult`
1825
1991
  :returns: The result of successively applying all the filters to the results
1826
- :rtype: :class:`~eodag.api.search_result.SearchResult`
1827
1992
  """
1828
1993
  search_criteria = kwargs.pop("search_criteria", {})
1829
1994
  for cruncher_name, cruncher_args in kwargs.items():
@@ -1839,7 +2004,6 @@ class EODataAccessGateway:
1839
2004
  by extent (i.e. bounding box).
1840
2005
 
1841
2006
  :param searches: List of eodag SearchResult
1842
- :type searches: list
1843
2007
  :returns: list of :class:`~eodag.api.search_result.SearchResult`
1844
2008
  """
1845
2009
  # Dict with extents as keys, each extent being defined by a str
@@ -1865,38 +2029,37 @@ class EODataAccessGateway:
1865
2029
  progress_callback: Optional[ProgressCallback] = None,
1866
2030
  wait: int = DEFAULT_DOWNLOAD_WAIT,
1867
2031
  timeout: int = DEFAULT_DOWNLOAD_TIMEOUT,
1868
- **kwargs: Any,
2032
+ **kwargs: Unpack[DownloadConf],
1869
2033
  ) -> List[str]:
1870
2034
  """Download all products resulting from a search.
1871
2035
 
1872
2036
  :param search_result: A collection of EO products resulting from a search
1873
- :type search_result: :class:`~eodag.api.search_result.SearchResult`
1874
2037
  :param downloaded_callback: (optional) A method or a callable object which takes
1875
2038
  as parameter the ``product``. You can use the base class
1876
2039
  :class:`~eodag.api.product.DownloadedCallback` and override
1877
2040
  its ``__call__`` method. Will be called each time a product
1878
2041
  finishes downloading
1879
- :type downloaded_callback: Callable[[:class:`~eodag.api.product._product.EOProduct`], None]
1880
- or None
1881
2042
  :param progress_callback: (optional) A method or a callable object
1882
2043
  which takes a current size and a maximum
1883
2044
  size as inputs and handle progress bar
1884
2045
  creation and update to give the user a
1885
2046
  feedback on the download progress
1886
- :type progress_callback: :class:`~eodag.utils.ProgressCallback` or None
1887
2047
  :param wait: (optional) If download fails, wait time in minutes between
1888
2048
  two download tries of the same product
1889
- :type wait: int
1890
2049
  :param timeout: (optional) If download fails, maximum time in minutes
1891
2050
  before stop retrying to download
1892
- :type timeout: int
1893
- :param kwargs: `outputs_prefix` (str), `extract` (bool), `delete_archive` (bool)
1894
- and `dl_url_params` (dict) can be provided as additional kwargs
1895
- and will override any other values defined in a configuration
1896
- file or with environment variables.
1897
- :type kwargs: Union[str, bool, dict]
2051
+ :param kwargs: Additional keyword arguments from the download plugin configuration class that can
2052
+ be provided to override any other values defined in a configuration file
2053
+ or with environment variables:
2054
+
2055
+ * ``output_dir`` - where to store downloaded products, as an absolute file path
2056
+ (Default: local temporary directory)
2057
+ * ``output_extension`` - downloaded file extension
2058
+ * ``extract`` - whether to extract the downloaded products, only applies to archived products
2059
+ * ``dl_url_params`` - additional parameters to pass over to the download url as an url parameter
2060
+ * ``delete_archive`` - whether to delete the downloaded archives
2061
+ * ``asset`` - regex filter to identify assets to download
1898
2062
  :returns: A collection of the absolute paths to the downloaded products
1899
- :rtype: list
1900
2063
  """
1901
2064
  paths = []
1902
2065
  if search_result:
@@ -1925,11 +2088,8 @@ class EODataAccessGateway:
1925
2088
  """Registers results of a search into a geojson file.
1926
2089
 
1927
2090
  :param search_result: A collection of EO products resulting from a search
1928
- :type search_result: :class:`~eodag.api.search_result.SearchResult`
1929
2091
  :param filename: (optional) The name of the file to generate
1930
- :type filename: str
1931
2092
  :returns: The name of the created file
1932
- :rtype: str
1933
2093
  """
1934
2094
  with open(filename, "w") as fh:
1935
2095
  geojson.dump(search_result, fh)
@@ -1940,9 +2100,7 @@ class EODataAccessGateway:
1940
2100
  """Loads results of a search from a geojson file.
1941
2101
 
1942
2102
  :param filename: A filename containing a search result encoded as a geojson
1943
- :type filename: str
1944
2103
  :returns: The search results encoded in `filename`
1945
- :rtype: :class:`~eodag.api.search_result.SearchResult`
1946
2104
  """
1947
2105
  with open(filename, "r") as fh:
1948
2106
  return SearchResult.from_geojson(geojson.load(fh))
@@ -1952,19 +2110,17 @@ class EODataAccessGateway:
1952
2110
  products with the information needed to download itself
1953
2111
 
1954
2112
  :param filename: A filename containing a search result encoded as a geojson
1955
- :type filename: str
1956
2113
  :returns: The search results encoded in `filename`
1957
- :rtype: :class:`~eodag.api.search_result.SearchResult`
1958
2114
  """
1959
2115
  products = self.deserialize(filename)
1960
2116
  for i, product in enumerate(products):
1961
2117
  if product.downloader is None:
2118
+ downloader = self._plugins_manager.get_download_plugin(product)
1962
2119
  auth = product.downloader_auth
1963
2120
  if auth is None:
1964
- auth = self._plugins_manager.get_auth_plugin(product.provider)
1965
- products[i].register_downloader(
1966
- self._plugins_manager.get_download_plugin(product), auth
1967
- )
2121
+ auth = self._plugins_manager.get_auth_plugin(downloader, product)
2122
+ products[i].register_downloader(downloader, auth)
2123
+
1968
2124
  return products
1969
2125
 
1970
2126
  @_deprecated(
@@ -1978,6 +2134,7 @@ class EODataAccessGateway:
1978
2134
  provider: Optional[str] = None,
1979
2135
  productType: Optional[str] = None,
1980
2136
  timeout: int = HTTP_REQ_TIMEOUT,
2137
+ ssl_verify: bool = True,
1981
2138
  **kwargs: Any,
1982
2139
  ) -> SearchResult:
1983
2140
  """Loads STAC items from a geojson file / STAC catalog or collection, and convert to SearchResult.
@@ -1986,22 +2143,14 @@ class EODataAccessGateway:
1986
2143
  the response content to an API request.
1987
2144
 
1988
2145
  :param filename: A filename containing features encoded as a geojson
1989
- :type filename: str
1990
2146
  :param recursive: (optional) Browse recursively in child nodes if True
1991
- :type recursive: bool
1992
2147
  :param max_connections: (optional) Maximum number of connections for HTTP requests
1993
- :type max_connections: int
1994
2148
  :param provider: (optional) Data provider
1995
- :type provider: str
1996
2149
  :param productType: (optional) Data product type
1997
- :type productType: str
1998
2150
  :param timeout: (optional) Timeout in seconds for each internal HTTP request
1999
- :type timeout: float
2000
2151
  :param kwargs: Parameters that will be stored in the result as
2001
2152
  search criteria
2002
- :type kwargs: Any
2003
2153
  :returns: The search results encoded in `filename`
2004
- :rtype: :class:`~eodag.api.search_result.SearchResult`
2005
2154
 
2006
2155
  .. deprecated:: 2.2.1
2007
2156
  Use the :class:`~eodag.plugins.search.static_stac_search.StaticStacSearch` search plugin instead.
@@ -2011,6 +2160,7 @@ class EODataAccessGateway:
2011
2160
  recursive=recursive,
2012
2161
  max_connections=max_connections,
2013
2162
  timeout=timeout,
2163
+ ssl_verify=ssl_verify,
2014
2164
  )
2015
2165
  feature_collection = geojson.FeatureCollection(features)
2016
2166
 
@@ -2027,12 +2177,14 @@ class EODataAccessGateway:
2027
2177
  )
2028
2178
  )
2029
2179
 
2030
- products, _ = self.search(productType=productType, provider=provider, **kwargs)
2180
+ search_result = self.search(
2181
+ productType=productType, provider=provider, **kwargs
2182
+ )
2031
2183
 
2032
2184
  # restore plugin._request
2033
2185
  plugin._request = plugin_request
2034
2186
 
2035
- return products
2187
+ return search_result
2036
2188
 
2037
2189
  def download(
2038
2190
  self,
@@ -2040,7 +2192,7 @@ class EODataAccessGateway:
2040
2192
  progress_callback: Optional[ProgressCallback] = None,
2041
2193
  wait: int = DEFAULT_DOWNLOAD_WAIT,
2042
2194
  timeout: int = DEFAULT_DOWNLOAD_TIMEOUT,
2043
- **kwargs: Any,
2195
+ **kwargs: Unpack[DownloadConf],
2044
2196
  ) -> str:
2045
2197
  """Download a single product.
2046
2198
 
@@ -2063,26 +2215,27 @@ class EODataAccessGateway:
2063
2215
  trying to download the product.
2064
2216
 
2065
2217
  :param product: The EO product to download
2066
- :type product: :class:`~eodag.api.product._product.EOProduct`
2067
2218
  :param progress_callback: (optional) A method or a callable object
2068
2219
  which takes a current size and a maximum
2069
2220
  size as inputs and handle progress bar
2070
2221
  creation and update to give the user a
2071
2222
  feedback on the download progress
2072
- :type progress_callback: :class:`~eodag.utils.ProgressCallback` or None
2073
2223
  :param wait: (optional) If download fails, wait time in minutes between
2074
2224
  two download tries
2075
- :type wait: int
2076
2225
  :param timeout: (optional) If download fails, maximum time in minutes
2077
2226
  before stop retrying to download
2078
- :type timeout: int
2079
- :param kwargs: `outputs_prefix` (str), `extract` (bool), `delete_archive` (bool)
2080
- and `dl_url_params` (dict) can be provided as additional kwargs
2081
- and will override any other values defined in a configuration
2082
- file or with environment variables.
2083
- :type kwargs: Union[str, bool, dict]
2227
+ :param kwargs: Additional keyword arguments from the download plugin configuration class that can
2228
+ be provided to override any other values defined in a configuration file
2229
+ or with environment variables:
2230
+
2231
+ * ``output_dir`` - where to store downloaded products, as an absolute file path
2232
+ (Default: local temporary directory)
2233
+ * ``output_extension`` - downloaded file extension
2234
+ * ``extract`` - whether to extract the downloaded products, only applies to archived products
2235
+ * ``dl_url_params`` - additional parameters to pass over to the download url as an url parameter
2236
+ * ``delete_archive`` - whether to delete the downloaded archives
2237
+ * ``asset`` - regex filter to identify assets to download
2084
2238
  :returns: The absolute path to the downloaded product in the local filesystem
2085
- :rtype: str
2086
2239
  :raises: :class:`~eodag.utils.exceptions.PluginImplementationError`
2087
2240
  :raises: :class:`RuntimeError`
2088
2241
  """
@@ -2098,146 +2251,77 @@ class EODataAccessGateway:
2098
2251
 
2099
2252
  def _setup_downloader(self, product: EOProduct) -> None:
2100
2253
  if product.downloader is None:
2254
+ downloader = self._plugins_manager.get_download_plugin(product)
2101
2255
  auth = product.downloader_auth
2102
2256
  if auth is None:
2103
- auth = self._plugins_manager.get_auth_plugin(product.provider)
2104
- product.register_downloader(
2105
- self._plugins_manager.get_download_plugin(product), auth
2106
- )
2257
+ auth = self._plugins_manager.get_auth_plugin(downloader, product)
2258
+ product.register_downloader(downloader, auth)
2107
2259
 
2108
2260
  def get_cruncher(self, name: str, **options: Any) -> Crunch:
2109
2261
  """Build a crunch plugin from a configuration
2110
2262
 
2111
2263
  :param name: The name of the cruncher to build
2112
- :type name: str
2113
2264
  :param options: The configuration options of the cruncher
2114
- :type options: dict
2115
2265
  :returns: The cruncher named ``name``
2116
- :rtype: :class:`~eodag.plugins.crunch.Crunch`
2117
2266
  """
2118
2267
  plugin_conf = {"name": name}
2119
2268
  plugin_conf.update({key.replace("-", "_"): val for key, val in options.items()})
2120
2269
  return self._plugins_manager.get_crunch_plugin(name, **plugin_conf)
2121
2270
 
2122
2271
  def list_queryables(
2123
- self,
2124
- provider: Optional[str] = None,
2125
- **kwargs: Any,
2272
+ self, provider: Optional[str] = None, **kwargs: Any
2126
2273
  ) -> Dict[str, Annotated[Any, FieldInfo]]:
2127
2274
  """Fetch the queryable properties for a given product type and/or provider.
2128
2275
 
2129
2276
  :param provider: (optional) The provider.
2130
- :type provider: str
2131
2277
  :param kwargs: additional filters for queryables (`productType` or other search
2132
2278
  arguments)
2133
- :type kwargs: Any
2279
+
2280
+ :raises UnsupportedProductType: If the specified product type is not available for the
2281
+ provider.
2282
+
2134
2283
  :returns: A dict containing the EODAG queryable properties, associating
2135
2284
  parameters to their annotated type
2136
- :rtype: Dict[str, Annotated[Any, FieldInfo]]
2137
2285
  """
2138
- # unknown product type
2139
2286
  available_product_types = [
2140
- pt["ID"] for pt in self.list_product_types(fetch_providers=False)
2287
+ pt["ID"]
2288
+ for pt in self.list_product_types(provider=provider, fetch_providers=False)
2141
2289
  ]
2142
- product_type = kwargs.get("productType", None)
2143
- if product_type is not None and product_type not in available_product_types:
2144
- self.fetch_product_types_list()
2145
-
2146
- # dictionary of the queryable properties of the providers supporting the given product type
2147
- providers_available_queryables: Dict[
2148
- str, Dict[str, Annotated[Any, FieldInfo]]
2149
- ] = dict()
2150
-
2151
- if provider is None and product_type is None:
2152
- return model_fields_to_annotated(CommonQueryables.model_fields)
2153
- elif provider is None:
2154
- for plugin in self._plugins_manager.get_search_plugins(
2155
- product_type, provider
2156
- ):
2157
- providers_available_queryables[plugin.provider] = self.list_queryables(
2158
- provider=plugin.provider, **kwargs
2159
- )
2290
+ product_type = kwargs.get("productType")
2160
2291
 
2161
- # return providers queryables intersection
2162
- queryables_keys: AbstractSet[str] = set()
2163
- for queryables in providers_available_queryables.values():
2164
- queryables_keys = (
2165
- queryables_keys & queryables.keys()
2166
- if queryables_keys
2167
- else queryables.keys()
2292
+ if product_type:
2293
+ try:
2294
+ kwargs["productType"] = product_type = self.get_product_type_from_alias(
2295
+ product_type
2168
2296
  )
2169
- return {
2170
- k: v
2171
- for k, v in providers_available_queryables.popitem()[1].items()
2172
- if k in queryables_keys
2173
- }
2297
+ except NoMatchingProductType as e:
2298
+ raise UnsupportedProductType(f"{product_type} is not available") from e
2174
2299
 
2175
- all_queryables = copy_deepcopy(
2176
- model_fields_to_annotated(Queryables.model_fields)
2177
- )
2300
+ if product_type and product_type not in available_product_types:
2301
+ self.fetch_product_types_list()
2178
2302
 
2179
- try:
2180
- plugin = next(
2181
- self._plugins_manager.get_search_plugins(product_type, provider)
2182
- )
2183
- except StopIteration:
2184
- # return default queryables if no plugin is found
2303
+ if not provider and not product_type:
2185
2304
  return model_fields_to_annotated(CommonQueryables.model_fields)
2186
2305
 
2187
- providers_available_queryables[plugin.provider] = dict()
2306
+ providers_queryables: Dict[str, Dict[str, Annotated[Any, FieldInfo]]] = {}
2188
2307
 
2189
- # unknown product type: try again after fetch_product_types_list()
2190
- if (
2191
- product_type
2192
- and product_type not in plugin.config.products.keys()
2193
- and provider is None
2194
- ):
2195
- raise UnsupportedProductType(product_type)
2196
- elif product_type and product_type not in plugin.config.products.keys():
2197
- raise UnsupportedProductType(
2198
- f"{product_type} is not available for provider {provider}"
2308
+ for plugin in self._plugins_manager.get_search_plugins(product_type, provider):
2309
+ if getattr(plugin.config, "need_auth", False) and (
2310
+ auth := self._plugins_manager.get_auth_plugin(plugin)
2311
+ ):
2312
+ plugin.auth = auth.authenticate()
2313
+ providers_queryables[plugin.provider] = plugin.list_queryables(
2314
+ filters=kwargs, product_type=product_type
2199
2315
  )
2200
2316
 
2201
- metadata_mapping = deepcopy(getattr(plugin.config, "metadata_mapping", {}))
2202
-
2203
- # product_type-specific metadata-mapping
2204
- metadata_mapping.update(
2205
- getattr(plugin.config, "products", {})
2206
- .get(product_type, {})
2207
- .get("metadata_mapping", {})
2208
- )
2209
-
2210
- # default values
2211
- default_values = deepcopy(
2212
- getattr(plugin.config, "products", {}).get(product_type, {})
2317
+ queryable_keys: Set[str] = set.intersection( # type: ignore
2318
+ *[set(q.keys()) for q in providers_queryables.values()]
2213
2319
  )
2214
- default_values.pop("metadata_mapping", None)
2215
- kwargs = dict(default_values, **kwargs)
2216
-
2217
- # remove not mapped parameters or non-queryables
2218
- for param in list(metadata_mapping.keys()):
2219
- if NOT_MAPPED in metadata_mapping[param] or not isinstance(
2220
- metadata_mapping[param], list
2221
- ):
2222
- del metadata_mapping[param]
2223
-
2224
- for key, value in all_queryables.items():
2225
- annotated_args = get_args(value)
2226
- if len(annotated_args) < 1:
2227
- continue
2228
- field_info = annotated_args[1]
2229
- if not isinstance(field_info, FieldInfo):
2230
- continue
2231
- if key in kwargs:
2232
- field_info.default = kwargs[key]
2233
- if field_info.is_required() or (
2234
- (field_info.alias or key) in metadata_mapping
2235
- ):
2236
- providers_available_queryables[plugin.provider][key] = value
2237
-
2238
- provider_queryables = plugin.discover_queryables(**kwargs) or dict()
2239
- # use EODAG configured queryables by default
2240
- provider_queryables.update(providers_available_queryables[provider])
2320
+ queryables = {
2321
+ k: v
2322
+ for k, v in list(providers_queryables.values())[0].items()
2323
+ if k in queryable_keys
2324
+ }
2241
2325
 
2242
2326
  # always keep at least CommonQueryables
2243
2327
  common_queryables = copy_deepcopy(CommonQueryables.model_fields)
@@ -2245,6 +2329,38 @@ class EODataAccessGateway:
2245
2329
  if key in kwargs:
2246
2330
  queryable.default = kwargs[key]
2247
2331
 
2248
- provider_queryables.update(model_fields_to_annotated(common_queryables))
2332
+ queryables.update(model_fields_to_annotated(common_queryables))
2333
+
2334
+ return queryables
2335
+
2336
+ def available_sortables(self) -> Dict[str, Optional[ProviderSortables]]:
2337
+ """For each provider, gives its available sortable parameter(s) and its maximum
2338
+ number of them if it supports the sorting feature, otherwise gives None.
2249
2339
 
2250
- return provider_queryables
2340
+ :returns: A dictionary with providers as keys and dictionary of sortable parameter(s) and
2341
+ its (their) maximum number as value(s).
2342
+ :raises: :class:`~eodag.utils.exceptions.UnsupportedProvider`
2343
+ """
2344
+ sortables: Dict[str, Optional[ProviderSortables]] = {}
2345
+ provider_search_plugins = self._plugins_manager.get_search_plugins()
2346
+ for provider_search_plugin in provider_search_plugins:
2347
+ provider = provider_search_plugin.provider
2348
+ if not hasattr(provider_search_plugin.config, "sort"):
2349
+ sortables[provider] = None
2350
+ continue
2351
+ sortable_params = list(
2352
+ provider_search_plugin.config.sort.get("sort_param_mapping", {}).keys()
2353
+ )
2354
+ if not provider_search_plugin.config.sort.get("max_sort_params"):
2355
+ sortables[provider] = {
2356
+ "sortables": sortable_params,
2357
+ "max_sort_params": None,
2358
+ }
2359
+ continue
2360
+ sortables[provider] = {
2361
+ "sortables": sortable_params,
2362
+ "max_sort_params": provider_search_plugin.config.sort[
2363
+ "max_sort_params"
2364
+ ],
2365
+ }
2366
+ return sortables