eodag 3.10.1__py3-none-any.whl → 4.0.0a2__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (75) hide show
  1. eodag/__init__.py +6 -1
  2. eodag/api/collection.py +353 -0
  3. eodag/api/core.py +606 -641
  4. eodag/api/product/__init__.py +3 -3
  5. eodag/api/product/_product.py +74 -56
  6. eodag/api/product/drivers/__init__.py +4 -46
  7. eodag/api/product/drivers/base.py +0 -28
  8. eodag/api/product/metadata_mapping.py +178 -216
  9. eodag/api/search_result.py +156 -15
  10. eodag/cli.py +83 -403
  11. eodag/config.py +81 -51
  12. eodag/plugins/apis/base.py +2 -2
  13. eodag/plugins/apis/ecmwf.py +36 -25
  14. eodag/plugins/apis/usgs.py +55 -40
  15. eodag/plugins/authentication/base.py +1 -3
  16. eodag/plugins/crunch/filter_date.py +3 -3
  17. eodag/plugins/crunch/filter_latest_intersect.py +2 -2
  18. eodag/plugins/crunch/filter_latest_tpl_name.py +1 -1
  19. eodag/plugins/download/aws.py +46 -42
  20. eodag/plugins/download/base.py +13 -14
  21. eodag/plugins/download/http.py +65 -65
  22. eodag/plugins/manager.py +28 -29
  23. eodag/plugins/search/__init__.py +6 -4
  24. eodag/plugins/search/base.py +131 -80
  25. eodag/plugins/search/build_search_result.py +245 -173
  26. eodag/plugins/search/cop_marine.py +87 -56
  27. eodag/plugins/search/csw.py +47 -37
  28. eodag/plugins/search/qssearch.py +653 -429
  29. eodag/plugins/search/stac_list_assets.py +1 -1
  30. eodag/plugins/search/static_stac_search.py +43 -44
  31. eodag/resources/{product_types.yml → collections.yml} +2594 -2453
  32. eodag/resources/ext_collections.json +1 -1
  33. eodag/resources/ext_product_types.json +1 -1
  34. eodag/resources/providers.yml +2706 -2733
  35. eodag/resources/stac_provider.yml +50 -92
  36. eodag/resources/user_conf_template.yml +9 -0
  37. eodag/types/__init__.py +2 -0
  38. eodag/types/queryables.py +70 -91
  39. eodag/types/search_args.py +1 -1
  40. eodag/utils/__init__.py +97 -21
  41. eodag/utils/dates.py +0 -12
  42. eodag/utils/exceptions.py +6 -6
  43. eodag/utils/free_text_search.py +3 -3
  44. eodag/utils/repr.py +2 -0
  45. {eodag-3.10.1.dist-info → eodag-4.0.0a2.dist-info}/METADATA +13 -99
  46. eodag-4.0.0a2.dist-info/RECORD +93 -0
  47. {eodag-3.10.1.dist-info → eodag-4.0.0a2.dist-info}/entry_points.txt +0 -4
  48. eodag/plugins/authentication/oauth.py +0 -60
  49. eodag/plugins/download/creodias_s3.py +0 -71
  50. eodag/plugins/download/s3rest.py +0 -351
  51. eodag/plugins/search/data_request_search.py +0 -565
  52. eodag/resources/stac.yml +0 -294
  53. eodag/resources/stac_api.yml +0 -2105
  54. eodag/rest/__init__.py +0 -24
  55. eodag/rest/cache.py +0 -70
  56. eodag/rest/config.py +0 -67
  57. eodag/rest/constants.py +0 -26
  58. eodag/rest/core.py +0 -764
  59. eodag/rest/errors.py +0 -210
  60. eodag/rest/server.py +0 -604
  61. eodag/rest/server.wsgi +0 -6
  62. eodag/rest/stac.py +0 -1032
  63. eodag/rest/templates/README +0 -1
  64. eodag/rest/types/__init__.py +0 -18
  65. eodag/rest/types/collections_search.py +0 -44
  66. eodag/rest/types/eodag_search.py +0 -386
  67. eodag/rest/types/queryables.py +0 -174
  68. eodag/rest/types/stac_search.py +0 -272
  69. eodag/rest/utils/__init__.py +0 -207
  70. eodag/rest/utils/cql_evaluate.py +0 -119
  71. eodag/rest/utils/rfc3339.py +0 -64
  72. eodag-3.10.1.dist-info/RECORD +0 -116
  73. {eodag-3.10.1.dist-info → eodag-4.0.0a2.dist-info}/WHEEL +0 -0
  74. {eodag-3.10.1.dist-info → eodag-4.0.0a2.dist-info}/licenses/LICENSE +0 -0
  75. {eodag-3.10.1.dist-info → eodag-4.0.0a2.dist-info}/top_level.txt +0 -0
eodag/api/core.py CHANGED
@@ -23,14 +23,17 @@ import os
23
23
  import re
24
24
  import shutil
25
25
  import tempfile
26
+ import warnings
27
+ from collections import deque
26
28
  from importlib.metadata import version
27
29
  from importlib.resources import files as res_files
28
- from operator import itemgetter
30
+ from operator import attrgetter, itemgetter
29
31
  from typing import TYPE_CHECKING, Any, Iterator, Optional, Union
30
32
 
31
33
  import geojson
32
- import yaml.parser
34
+ import yaml
33
35
 
36
+ from eodag.api.collection import Collection, CollectionsDict, CollectionsList
34
37
  from eodag.api.product.metadata_mapping import (
35
38
  NOT_AVAILABLE,
36
39
  mtd_cfg_as_conversion_and_querypath,
@@ -41,7 +44,7 @@ from eodag.config import (
41
44
  PluginConfig,
42
45
  SimpleYamlProxyConfig,
43
46
  credentials_in_auth,
44
- get_ext_product_types_conf,
47
+ get_ext_collections_conf,
45
48
  load_default_config,
46
49
  load_stac_provider_config,
47
50
  load_yml_config,
@@ -53,7 +56,10 @@ from eodag.config import (
53
56
  )
54
57
  from eodag.plugins.manager import PluginManager
55
58
  from eodag.plugins.search import PreparedSearch
56
- from eodag.plugins.search.build_search_result import MeteoblueSearch
59
+ from eodag.plugins.search.build_search_result import (
60
+ ALLOWED_KEYWORDS as ECMWF_ALLOWED_KEYWORDS,
61
+ )
62
+ from eodag.plugins.search.build_search_result import ECMWF_PREFIX, MeteoblueSearch
57
63
  from eodag.plugins.search.qssearch import PostJsonSearch
58
64
  from eodag.types import model_fields_to_annotated
59
65
  from eodag.types.queryables import CommonQueryables, Queryables, QueryablesDict
@@ -63,10 +69,8 @@ from eodag.utils import (
63
69
  DEFAULT_ITEMS_PER_PAGE,
64
70
  DEFAULT_MAX_ITEMS_PER_PAGE,
65
71
  DEFAULT_PAGE,
66
- GENERIC_PRODUCT_TYPE,
72
+ GENERIC_COLLECTION,
67
73
  GENERIC_STAC_PROVIDER,
68
- HTTP_REQ_TIMEOUT,
69
- MockResponse,
70
74
  _deprecated,
71
75
  get_geometry_from_various,
72
76
  makedirs,
@@ -78,11 +82,12 @@ from eodag.utils.dates import rfc3339_str_to_datetime
78
82
  from eodag.utils.env import is_env_var_true
79
83
  from eodag.utils.exceptions import (
80
84
  AuthenticationError,
81
- NoMatchingProductType,
85
+ NoMatchingCollection,
82
86
  PluginImplementationError,
83
87
  RequestError,
84
- UnsupportedProductType,
88
+ UnsupportedCollection,
85
89
  UnsupportedProvider,
90
+ ValidationError,
86
91
  )
87
92
  from eodag.utils.free_text_search import compile_free_text_query
88
93
  from eodag.utils.stac_reader import fetch_stac_items
@@ -114,10 +119,11 @@ class EODataAccessGateway:
114
119
  user_conf_file_path: Optional[str] = None,
115
120
  locations_conf_path: Optional[str] = None,
116
121
  ) -> None:
117
- product_types_config_path = os.getenv("EODAG_PRODUCT_TYPES_CFG_FILE") or str(
118
- res_files("eodag") / "resources" / "product_types.yml"
122
+ collections_config_path = os.getenv("EODAG_COLLECTIONS_CFG_FILE") or str(
123
+ res_files("eodag") / "resources" / "collections.yml"
119
124
  )
120
- self.product_types_config = SimpleYamlProxyConfig(product_types_config_path)
125
+ collections_config_dict = SimpleYamlProxyConfig(collections_config_path).source
126
+ self.collections_config = self._collections_config_init(collections_config_dict)
121
127
  self.providers_config = load_default_config()
122
128
 
123
129
  env_var_cfg_dir = "EODAG_CFG_DIR"
@@ -169,8 +175,8 @@ class EODataAccessGateway:
169
175
  share_credentials(self.providers_config)
170
176
 
171
177
  # init updated providers conf
172
- strict_mode = is_env_var_true("EODAG_STRICT_PRODUCT_TYPES")
173
- available_product_types = set(self.product_types_config.source.keys())
178
+ strict_mode = is_env_var_true("EODAG_STRICT_COLLECTIONS")
179
+ available_collections = set(self.collections_config.keys())
174
180
 
175
181
  for provider in self.providers_config.keys():
176
182
  provider_config_init(
@@ -178,11 +184,9 @@ class EODataAccessGateway:
178
184
  load_stac_provider_config(),
179
185
  )
180
186
 
181
- self._sync_provider_product_types(
182
- provider, available_product_types, strict_mode
187
+ self._sync_provider_collections(
188
+ provider, available_collections, strict_mode
183
189
  )
184
- # init product types configuration
185
- self._product_types_config_init()
186
190
 
187
191
  # re-build _plugins_manager using up-to-date providers_config
188
192
  self._plugins_manager.rebuild(self.providers_config)
@@ -225,26 +229,35 @@ class EODataAccessGateway:
225
229
  )
226
230
  self.set_locations_conf(locations_conf_path)
227
231
 
228
- def _product_types_config_init(self) -> None:
229
- """Initialize product types configuration."""
230
- for pt_id, pd_dict in self.product_types_config.source.items():
231
- self.product_types_config.source[pt_id].setdefault("_id", pt_id)
232
+ def _collections_config_init(
233
+ self, collections_config_dict: dict[str, Any]
234
+ ) -> CollectionsDict:
235
+ """Initialize collections configuration.
232
236
 
233
- def _sync_provider_product_types(
237
+ :param collections_config_dict: The collections config as a dictionary
238
+ """
239
+ # Turn the collections config from a dict into a CollectionsDict() object
240
+ collections = [
241
+ Collection.create_with_dag(self, id=col, **col_f)
242
+ for col, col_f in collections_config_dict.items()
243
+ ]
244
+ return CollectionsDict(collections)
245
+
246
+ def _sync_provider_collections(
234
247
  self,
235
248
  provider: str,
236
- available_product_types: set[str],
249
+ available_collections: set[str],
237
250
  strict_mode: bool,
238
251
  ) -> None:
239
252
  """
240
- Synchronize product types for a provider based on strict or permissive mode.
253
+ Synchronize collections for a provider based on strict or permissive mode.
241
254
 
242
- In strict mode, removes product types not in available_product_types.
243
- In permissive mode, adds empty product type configs for missing types.
255
+ In strict mode, removes collections not in available_collections.
256
+ In permissive mode, adds empty collection configs for missing types.
244
257
 
245
- :param provider: The provider name whose product types should be synchronized.
246
- :param available_product_types: The set of available product type IDs.
247
- :param strict_mode: If True, remove unknown product types; if False, add empty configs for them.
258
+ :param provider: The provider name whose collections should be synchronized.
259
+ :param available_collections: The set of available collection IDs.
260
+ :param strict_mode: If True, remove unknown collections; if False, add empty configs for them.
248
261
  :returns: None
249
262
  """
250
263
  provider_products = self.providers_config[provider].products
@@ -252,33 +265,30 @@ class EODataAccessGateway:
252
265
  products_to_add: list[str] = []
253
266
 
254
267
  for product_id in provider_products:
255
- if product_id == GENERIC_PRODUCT_TYPE:
268
+ if product_id == GENERIC_COLLECTION:
256
269
  continue
257
270
 
258
- if product_id not in available_product_types:
271
+ if product_id not in available_collections:
259
272
  if strict_mode:
260
273
  products_to_remove.append(product_id)
261
274
  continue
262
275
 
263
- empty_product = {
264
- "title": product_id,
265
- "abstract": NOT_AVAILABLE,
266
- }
267
- self.product_types_config.source[
268
- product_id
269
- ] = empty_product # will update available_product_types
276
+ empty_product = Collection.create_with_dag(
277
+ self, id=product_id, title=product_id, description=NOT_AVAILABLE
278
+ )
279
+ self.collections_config[product_id] = empty_product
270
280
  products_to_add.append(product_id)
271
281
 
272
282
  if products_to_add:
273
283
  logger.debug(
274
- "Product types permissive mode, %s added (provider %s)",
284
+ "Collections permissive mode, %s added (provider %s)",
275
285
  ", ".join(products_to_add),
276
286
  provider,
277
287
  )
278
288
 
279
289
  if products_to_remove:
280
290
  logger.debug(
281
- "Product types strict mode, ignoring %s (provider %s)",
291
+ "Collections strict mode, ignoring %s (provider %s)",
282
292
  ", ".join(products_to_remove),
283
293
  provider,
284
294
  )
@@ -357,9 +367,9 @@ class EODataAccessGateway:
357
367
  self.providers_config[provider],
358
368
  load_stac_provider_config(),
359
369
  )
360
- setattr(self.providers_config[provider], "product_types_fetched", False)
370
+ setattr(self.providers_config[provider], "collections_fetched", False)
361
371
  # re-create _plugins_manager using up-to-date providers_config
362
- self._plugins_manager.build_product_type_to_provider_config_map()
372
+ self._plugins_manager.build_collection_to_provider_config_map()
363
373
 
364
374
  def add_provider(
365
375
  self,
@@ -368,7 +378,7 @@ class EODataAccessGateway:
368
378
  priority: Optional[int] = None,
369
379
  search: dict[str, Any] = {"type": "StacSearch"},
370
380
  products: dict[str, Any] = {
371
- GENERIC_PRODUCT_TYPE: {"productType": "{productType}"}
381
+ GENERIC_COLLECTION: {"_collection": "{collection}"}
372
382
  },
373
383
  download: dict[str, Any] = {"type": "HTTPDownload", "auth_error_code": 401},
374
384
  **kwargs: dict[str, Any],
@@ -379,14 +389,14 @@ class EODataAccessGateway:
379
389
  updated (not replaced), with user provided ones:
380
390
 
381
391
  * ``search`` : ``{"type": "StacSearch"}``
382
- * ``products`` : ``{"GENERIC_PRODUCT_TYPE": {"productType": "{productType}"}}``
392
+ * ``products`` : ``{"GENERIC_COLLECTION": {"_collection": "{collection}"}}``
383
393
  * ``download`` : ``{"type": "HTTPDownload", "auth_error_code": 401}``
384
394
 
385
395
  :param name: Name of provider
386
396
  :param url: Provider url, also used as ``search["api_endpoint"]`` if not defined
387
397
  :param priority: Provider priority. If None, provider will be set as preferred (highest priority)
388
398
  :param search: Search :class:`~eodag.config.PluginConfig` mapping
389
- :param products: Provider product types mapping
399
+ :param products: Provider collections mapping
390
400
  :param download: Download :class:`~eodag.config.PluginConfig` mapping
391
401
  :param kwargs: Additional :class:`~eodag.config.ProviderConfig` mapping
392
402
  """
@@ -395,7 +405,7 @@ class EODataAccessGateway:
395
405
  "url": url,
396
406
  "search": {"type": "StacSearch", **search},
397
407
  "products": {
398
- GENERIC_PRODUCT_TYPE: {"productType": "{productType}"},
408
+ GENERIC_COLLECTION: {"_collection": "{collection}"},
399
409
  **products,
400
410
  },
401
411
  "download": {
@@ -541,23 +551,23 @@ class EODataAccessGateway:
541
551
  )
542
552
  self.locations_config = []
543
553
 
544
- def list_product_types(
554
+ def list_collections(
545
555
  self, provider: Optional[str] = None, fetch_providers: bool = True
546
- ) -> list[dict[str, Any]]:
547
- """Lists supported product types.
556
+ ) -> CollectionsList:
557
+ """Lists supported collections.
548
558
 
549
559
  :param provider: (optional) The name of a provider that must support the product
550
560
  types we are about to list
551
561
  :param fetch_providers: (optional) Whether to fetch providers for new product
552
562
  types or not
553
- :returns: The list of the product types that can be accessed using eodag.
563
+ :returns: The list of the collections that can be accessed using eodag.
554
564
  :raises: :class:`~eodag.utils.exceptions.UnsupportedProvider`
555
565
  """
556
566
  if fetch_providers:
557
- # First, update product types list if possible
558
- self.fetch_product_types_list(provider=provider)
567
+ # First, update collections list if possible
568
+ self.fetch_collections_list(provider=provider)
559
569
 
560
- product_types: list[dict[str, Any]] = []
570
+ collections: CollectionsList = CollectionsList([])
561
571
 
562
572
  providers_configs = (
563
573
  list(self.providers_config.values())
@@ -575,31 +585,29 @@ class EODataAccessGateway:
575
585
  )
576
586
 
577
587
  for p in providers_configs:
578
- for product_type_id in p.products: # type: ignore
579
- if product_type_id == GENERIC_PRODUCT_TYPE:
588
+ for collection_id in p.products:
589
+ if collection_id == GENERIC_COLLECTION:
580
590
  continue
581
591
 
582
- config = self.product_types_config[product_type_id]
583
- if "alias" in config:
584
- product_type_id = config["alias"]
585
- product_type = {"ID": product_type_id, **config}
586
-
587
- if product_type not in product_types:
588
- product_types.append(product_type)
592
+ if (
593
+ collection := self.collections_config[collection_id]
594
+ ) not in collections:
595
+ collections.append(collection)
589
596
 
590
- # Return the product_types sorted in lexicographic order of their ID
591
- return sorted(product_types, key=itemgetter("ID"))
597
+ # Return the collections sorted in lexicographic order of their id
598
+ collections.sort(key=attrgetter("id"))
599
+ return collections
592
600
 
593
- def fetch_product_types_list(self, provider: Optional[str] = None) -> None:
594
- """Fetch product types list and update if needed.
601
+ def fetch_collections_list(self, provider: Optional[str] = None) -> None:
602
+ """Fetch collections list and update if needed.
595
603
 
596
- If strict mode is enabled (by setting the ``EODAG_STRICT_PRODUCT_TYPES`` environment variable
597
- to a truthy value), this method will not fetch or update product types and will return immediately.
604
+ If strict mode is enabled (by setting the ``EODAG_STRICT_COLLECTIONS`` environment variable
605
+ to a truthy value), this method will not fetch or update collections and will return immediately.
598
606
 
599
- :param provider: The name of a provider or provider-group for which product types
607
+ :param provider: The name of a provider or provider-group for which collections
600
608
  list should be updated. Defaults to all providers (None value).
601
609
  """
602
- strict_mode = is_env_var_true("EODAG_STRICT_PRODUCT_TYPES")
610
+ strict_mode = is_env_var_true("EODAG_STRICT_COLLECTIONS")
603
611
  if strict_mode:
604
612
  return
605
613
 
@@ -613,7 +621,7 @@ class EODataAccessGateway:
613
621
  ]
614
622
  if providers_to_fetch:
615
623
  logger.info(
616
- f"Fetch product types for {provider} group: {', '.join(providers_to_fetch)}"
624
+ f"Fetch collections for {provider} group: {', '.join(providers_to_fetch)}"
617
625
  )
618
626
  else:
619
627
  return None
@@ -622,7 +630,7 @@ class EODataAccessGateway:
622
630
 
623
631
  # providers discovery confs that are fetchable
624
632
  providers_discovery_configs_fetchable: dict[str, Any] = {}
625
- # check if any provider has not already been fetched for product types
633
+ # check if any provider has not already been fetched for collections
626
634
  already_fetched = True
627
635
  for provider_to_fetch in providers_to_fetch:
628
636
  provider_config = self.providers_config[provider_to_fetch]
@@ -633,45 +641,43 @@ class EODataAccessGateway:
633
641
  provider_search_config = provider_config.api
634
642
  else:
635
643
  continue
636
- discovery_conf = getattr(
637
- provider_search_config, "discover_product_types", {}
638
- )
644
+ discovery_conf = getattr(provider_search_config, "discover_collections", {})
639
645
  if discovery_conf.get("fetch_url"):
640
646
  providers_discovery_configs_fetchable[
641
647
  provider_to_fetch
642
648
  ] = discovery_conf
643
- if not getattr(provider_config, "product_types_fetched", False):
649
+ if not getattr(provider_config, "collections_fetched", False):
644
650
  already_fetched = False
645
651
 
646
652
  if not already_fetched:
647
- # get ext_product_types conf
648
- ext_product_types_cfg_file = os.getenv("EODAG_EXT_PRODUCT_TYPES_CFG_FILE")
649
- if ext_product_types_cfg_file is not None:
650
- ext_product_types_conf = get_ext_product_types_conf(
651
- ext_product_types_cfg_file
653
+ # get ext_collections conf
654
+ ext_collections_cfg_file = os.getenv("EODAG_EXT_COLLECTIONS_CFG_FILE")
655
+ if ext_collections_cfg_file is not None:
656
+ ext_collections_conf = get_ext_collections_conf(
657
+ ext_collections_cfg_file
652
658
  )
653
659
  else:
654
- ext_product_types_conf = get_ext_product_types_conf()
660
+ ext_collections_conf = get_ext_collections_conf()
655
661
 
656
- if not ext_product_types_conf:
657
- # empty ext_product_types conf
658
- ext_product_types_conf = (
659
- self.discover_product_types(provider=provider) or {}
662
+ if not ext_collections_conf:
663
+ # empty ext_collections conf
664
+ ext_collections_conf = (
665
+ self.discover_collections(provider=provider) or {}
660
666
  )
661
667
 
662
- # update eodag product types list with new conf
663
- self.update_product_types_list(ext_product_types_conf)
668
+ # update eodag collections list with new conf
669
+ self.update_collections_list(ext_collections_conf)
664
670
 
665
671
  # Compare current provider with default one to see if it has been modified
666
- # and product types list would need to be fetched
672
+ # and collections list would need to be fetched
667
673
 
668
- # get ext_product_types conf for user modified providers
674
+ # get ext_collections conf for user modified providers
669
675
  default_providers_config = load_default_config()
670
676
  for (
671
677
  provider,
672
678
  user_discovery_conf,
673
679
  ) in providers_discovery_configs_fetchable.items():
674
- # default discover_product_types conf
680
+ # default discover_collections conf
675
681
  if provider in default_providers_config:
676
682
  default_provider_config = default_providers_config[provider]
677
683
  if hasattr(default_provider_config, "search"):
@@ -681,7 +687,7 @@ class EODataAccessGateway:
681
687
  else:
682
688
  continue
683
689
  default_discovery_conf = getattr(
684
- default_provider_search_config, "discover_product_types", {}
690
+ default_provider_search_config, "discover_collections", {}
685
691
  )
686
692
  # compare confs
687
693
  if default_discovery_conf["result_type"] == "json" and isinstance(
@@ -696,22 +702,22 @@ class EODataAccessGateway:
696
702
  },
697
703
  **mtd_cfg_as_conversion_and_querypath(
698
704
  dict(
699
- generic_product_type_id=default_discovery_conf[
700
- "generic_product_type_id"
705
+ generic_collection_id=default_discovery_conf[
706
+ "generic_collection_id"
701
707
  ]
702
708
  )
703
709
  ),
704
710
  **dict(
705
- generic_product_type_parsable_properties=mtd_cfg_as_conversion_and_querypath(
711
+ generic_collection_parsable_properties=mtd_cfg_as_conversion_and_querypath(
706
712
  default_discovery_conf[
707
- "generic_product_type_parsable_properties"
713
+ "generic_collection_parsable_properties"
708
714
  ]
709
715
  )
710
716
  ),
711
717
  **dict(
712
- generic_product_type_parsable_metadata=mtd_cfg_as_conversion_and_querypath(
718
+ generic_collection_parsable_metadata=mtd_cfg_as_conversion_and_querypath(
713
719
  default_discovery_conf[
714
- "generic_product_type_parsable_metadata"
720
+ "generic_collection_parsable_metadata"
715
721
  ]
716
722
  )
717
723
  ),
@@ -723,33 +729,33 @@ class EODataAccessGateway:
723
729
  or user_discovery_conf == default_discovery_conf_parsed
724
730
  ) and (
725
731
  not default_discovery_conf.get("fetch_url")
726
- or "ext_product_types_conf" not in locals()
727
- or "ext_product_types_conf" in locals()
732
+ or "ext_collections_conf" not in locals()
733
+ or "ext_collections_conf" in locals()
728
734
  and (
729
- provider in ext_product_types_conf
730
- or len(ext_product_types_conf.keys()) == 0
735
+ provider in ext_collections_conf
736
+ or len(ext_collections_conf.keys()) == 0
731
737
  )
732
738
  ):
733
739
  continue
734
740
  # providers not skipped here should be user-modified
735
- # or not in ext_product_types_conf (if eodag system conf != eodag conf used for ext_product_types_conf)
741
+ # or not in ext_collections_conf (if eodag system conf != eodag conf used for ext_collections_conf)
736
742
 
737
743
  if not already_fetched:
738
- # discover product types for user configured provider
739
- provider_ext_product_types_conf = (
740
- self.discover_product_types(provider=provider) or {}
744
+ # discover collections for user configured provider
745
+ provider_ext_collections_conf = (
746
+ self.discover_collections(provider=provider) or {}
741
747
  )
742
- # update eodag product types list with new conf
743
- self.update_product_types_list(provider_ext_product_types_conf)
748
+ # update eodag collections list with new conf
749
+ self.update_collections_list(provider_ext_collections_conf)
744
750
 
745
- def discover_product_types(
751
+ def discover_collections(
746
752
  self, provider: Optional[str] = None
747
753
  ) -> Optional[dict[str, Any]]:
748
- """Fetch providers for product types
754
+ """Fetch providers for collections
749
755
 
750
756
  :param provider: The name of a provider or provider-group to fetch. Defaults to
751
757
  all providers (None value).
752
- :returns: external product types configuration
758
+ :returns: external collections configuration
753
759
  """
754
760
  grouped_providers = [
755
761
  p
@@ -758,13 +764,13 @@ class EODataAccessGateway:
758
764
  ]
759
765
  if provider and provider not in self.providers_config and grouped_providers:
760
766
  logger.info(
761
- f"Discover product types for {provider} group: {', '.join(grouped_providers)}"
767
+ f"Discover collections for {provider} group: {', '.join(grouped_providers)}"
762
768
  )
763
769
  elif provider and provider not in self.providers_config:
764
770
  raise UnsupportedProvider(
765
771
  f"The requested provider is not (yet) supported: {provider}"
766
772
  )
767
- ext_product_types_conf: dict[str, Any] = {}
773
+ ext_collections_conf: dict[str, Any] = {}
768
774
  providers_to_fetch = [
769
775
  p
770
776
  for p in (
@@ -785,14 +791,14 @@ class EODataAccessGateway:
785
791
  search_plugin_config = self.providers_config[provider].api
786
792
  else:
787
793
  return None
788
- if getattr(search_plugin_config, "discover_product_types", {}).get(
794
+ if getattr(search_plugin_config, "discover_collections", {}).get(
789
795
  "fetch_url", None
790
796
  ):
791
797
  search_plugin: Union[Search, Api] = next(
792
798
  self._plugins_manager.get_search_plugins(provider=provider)
793
799
  )
794
800
  # check after plugin init if still fetchable
795
- if not getattr(search_plugin.config, "discover_product_types", {}).get(
801
+ if not getattr(search_plugin.config, "discover_collections", {}).get(
796
802
  "fetch_url"
797
803
  ):
798
804
  continue
@@ -806,26 +812,26 @@ class EODataAccessGateway:
806
812
  kwargs["auth"] = auth
807
813
  else:
808
814
  logger.debug(
809
- f"Could not authenticate on {provider} for product types discovery"
815
+ f"Could not authenticate on {provider} for collections discovery"
810
816
  )
811
- ext_product_types_conf[provider] = None
817
+ ext_collections_conf[provider] = None
812
818
  continue
813
819
 
814
- ext_product_types_conf[provider] = search_plugin.discover_product_types(
820
+ ext_collections_conf[provider] = search_plugin.discover_collections(
815
821
  **kwargs
816
822
  )
817
823
 
818
- return sort_dict(ext_product_types_conf)
824
+ return sort_dict(ext_collections_conf)
819
825
 
820
- def update_product_types_list(
821
- self, ext_product_types_conf: dict[str, Optional[dict[str, dict[str, Any]]]]
826
+ def update_collections_list(
827
+ self, ext_collections_conf: dict[str, Optional[dict[str, dict[str, Any]]]]
822
828
  ) -> None:
823
- """Update eodag product types list
829
+ """Update eodag collections list
824
830
 
825
- :param ext_product_types_conf: external product types configuration
831
+ :param ext_collections_conf: external collections configuration
826
832
  """
827
- for provider, new_product_types_conf in ext_product_types_conf.items():
828
- if new_product_types_conf and provider in self.providers_config:
833
+ for provider, new_collections_conf in ext_collections_conf.items():
834
+ if new_collections_conf and provider in self.providers_config:
829
835
  try:
830
836
  search_plugin_config = getattr(
831
837
  self.providers_config[provider], "search", None
@@ -833,77 +839,108 @@ class EODataAccessGateway:
833
839
  if search_plugin_config is None:
834
840
  continue
835
841
  if not getattr(
836
- search_plugin_config, "discover_product_types", {}
842
+ search_plugin_config, "discover_collections", {}
837
843
  ).get("fetch_url"):
838
- # conf has been updated and provider product types are no more discoverable
844
+ # conf has been updated and provider collections are no more discoverable
839
845
  continue
840
846
  provider_products_config = (
841
847
  self.providers_config[provider].products or {}
842
848
  )
843
849
  except UnsupportedProvider:
844
850
  logger.debug(
845
- "Ignoring external product types for unknown provider %s",
851
+ "Ignoring external collections for unknown provider %s",
846
852
  provider,
847
853
  )
848
854
  continue
849
- new_product_types: list[str] = []
855
+ new_collections: list[str] = []
856
+ bad_formatted_col_count = 0
850
857
  for (
851
- new_product_type,
852
- new_product_type_conf,
853
- ) in new_product_types_conf["providers_config"].items():
854
- if new_product_type not in provider_products_config:
855
- for existing_product_type in provider_products_config.copy():
858
+ new_collection,
859
+ new_collection_conf,
860
+ ) in new_collections_conf["providers_config"].items():
861
+ if new_collection not in provider_products_config:
862
+ for existing_collection in provider_products_config.copy():
856
863
  # compare parsed extracted conf (without metadata_mapping entry)
857
864
  unparsable_keys = (
858
- search_plugin_config.discover_product_types.get(
859
- "generic_product_type_unparsable_properties", {}
865
+ search_plugin_config.discover_collections.get(
866
+ "generic_collection_unparsable_properties", {}
860
867
  ).keys()
861
868
  )
862
- new_parsed_product_types_conf = {
869
+ new_parsed_collections_conf = {
863
870
  k: v
864
- for k, v in new_product_type_conf.items()
871
+ for k, v in new_collection_conf.items()
865
872
  if k not in unparsable_keys
866
873
  }
867
874
  if (
868
- new_parsed_product_types_conf.items()
869
- <= provider_products_config[
870
- existing_product_type
871
- ].items()
875
+ new_parsed_collections_conf.items()
876
+ <= provider_products_config[existing_collection].items()
872
877
  ):
873
- # new_product_types_conf is a subset on an existing conf
878
+ # new_collections_conf is a subset on an existing conf
874
879
  break
875
880
  else:
876
- # new_product_type_conf does not already exist, append it
877
- # to provider_products_config
878
- provider_products_config[
879
- new_product_type
880
- ] = new_product_type_conf
881
- # to self.product_types_config
882
- self.product_types_config.source.update(
883
- {
884
- new_product_type: {"_id": new_product_type}
885
- | new_product_types_conf["product_types_config"][
886
- new_product_type
887
- ]
881
+ try:
882
+ # new_collection_conf does not already exist, append it
883
+ # to self.collections_config
884
+ self.collections_config[
885
+ new_collection
886
+ ] = Collection.create_with_dag(
887
+ self,
888
+ id=new_collection,
889
+ **new_collections_conf["collections_config"][
890
+ new_collection
891
+ ],
892
+ )
893
+ except ValidationError:
894
+ # skip collection if there is a problem with its id (missing or not a string)
895
+ logger.debug(
896
+ (
897
+ "Collection %s has been pruned on provider %s "
898
+ "because its id was incorrectly parsed for eodag"
899
+ ),
900
+ new_collection,
901
+ provider,
902
+ )
903
+ else:
904
+ # to provider_products_config
905
+ provider_products_config[
906
+ new_collection
907
+ ] = new_collection_conf
908
+ ext_collections_conf[provider] = new_collections_conf
909
+ new_collections.append(new_collection)
910
+ # increase the increment if the new collection had
911
+ # bad formatted attributes in the external config
912
+ dumped_collection = self.collections_config[
913
+ new_collection
914
+ ].model_dump()
915
+ dumped_ext_conf_col = {
916
+ **dumped_collection,
917
+ **new_collections_conf["collections_config"][
918
+ new_collection
919
+ ],
888
920
  }
889
- )
890
- ext_product_types_conf[provider] = new_product_types_conf
891
- new_product_types.append(new_product_type)
892
- if new_product_types:
921
+ if dumped_ext_conf_col != dumped_collection:
922
+ bad_formatted_col_count += 1
923
+ if new_collections:
893
924
  logger.debug(
894
- f"Added {len(new_product_types)} product types for {provider}"
925
+ "Added %s collections for %s", len(new_collections), provider
895
926
  )
927
+ if bad_formatted_col_count > 0:
928
+ logger.debug(
929
+ "bad formatted attributes skipped for %s collection(s) on %s",
930
+ bad_formatted_col_count,
931
+ provider,
932
+ )
896
933
 
897
934
  elif provider not in self.providers_config:
898
935
  # unknown provider
899
936
  continue
900
- self.providers_config[provider].product_types_fetched = True
937
+ self.providers_config[provider].collections_fetched = True
901
938
 
902
939
  # re-create _plugins_manager using up-to-date providers_config
903
- self._plugins_manager.build_product_type_to_provider_config_map()
940
+ self._plugins_manager.build_collection_to_provider_config_map()
904
941
 
905
942
  def available_providers(
906
- self, product_type: Optional[str] = None, by_group: bool = False
943
+ self, collection: Optional[str] = None, by_group: bool = False
907
944
  ) -> list[str]:
908
945
  """Gives the sorted list of the available providers or groups
909
946
 
@@ -911,17 +948,17 @@ class EODataAccessGateway:
911
948
  and then alphabetically in ascending order for providers or groups with the same
912
949
  priority level.
913
950
 
914
- :param product_type: (optional) Only list providers configured for this product_type
951
+ :param collection: (optional) Only list providers configured for this collection
915
952
  :param by_group: (optional) If set to True, list groups when available instead
916
953
  of providers, mixed with other providers
917
954
  :returns: the sorted list of the available providers or groups
918
955
  """
919
956
 
920
- if product_type:
957
+ if collection:
921
958
  providers = [
922
959
  (v.group if by_group and hasattr(v, "group") else k, v.priority)
923
960
  for k, v in self.providers_config.items()
924
- if product_type in getattr(v, "products", {}).keys()
961
+ if collection in getattr(v, "products", {}).keys()
925
962
  ]
926
963
  else:
927
964
  providers = [
@@ -943,97 +980,103 @@ class EODataAccessGateway:
943
980
  # Return only the names of the providers or groups
944
981
  return [name for name, _ in providers]
945
982
 
946
- def get_product_type_from_alias(self, alias_or_id: str) -> str:
947
- """Return the ID of a product type by either its ID or alias
983
+ def get_collection_from_alias(self, alias_or_id: str) -> str:
984
+ """Return the id of a collection by either its id or alias
948
985
 
949
- :param alias_or_id: Alias of the product type. If an existing ID is given, this
986
+ :param alias_or_id: Alias of the collection. If an existing id is given, this
950
987
  method will directly return the given value.
951
- :returns: Internal name of the product type.
988
+ :returns: Internal name of the collection.
952
989
  """
953
- product_types = [
954
- k
955
- for k, v in self.product_types_config.items()
956
- if v.get("alias") == alias_or_id
990
+ collections = [
991
+ k for k, v in self.collections_config.items() if v.alias == alias_or_id
957
992
  ]
958
993
 
959
- if len(product_types) > 1:
960
- raise NoMatchingProductType(
961
- f"Too many matching product types for alias {alias_or_id}: {product_types}"
994
+ if len(collections) > 1:
995
+ raise NoMatchingCollection(
996
+ f"Too many matching collections for alias {alias_or_id}: {collections}"
962
997
  )
963
998
 
964
- if len(product_types) == 0:
965
- if alias_or_id in self.product_types_config:
999
+ if len(collections) == 0:
1000
+ if alias_or_id in self.collections_config:
966
1001
  return alias_or_id
967
1002
  else:
968
- raise NoMatchingProductType(
969
- f"Could not find product type from alias or ID {alias_or_id}"
1003
+ raise NoMatchingCollection(
1004
+ f"Could not find collection from alias or id {alias_or_id}"
970
1005
  )
971
1006
 
972
- return product_types[0]
1007
+ return collections[0]
973
1008
 
974
- def get_alias_from_product_type(self, product_type: str) -> str:
975
- """Return the alias of a product type by its ID. If no alias was defined for the
976
- given product type, its ID is returned instead.
1009
+ def get_alias_from_collection(self, collection: str) -> str:
1010
+ """Return the alias of a collection by its id. If no alias was defined for the
1011
+ given collection, its id is returned instead.
977
1012
 
978
- :param product_type: product type ID
979
- :returns: Alias of the product type or its ID if no alias has been defined for it.
1013
+ :param collection: collection id
1014
+ :returns: Alias of the collection or its id if no alias has been defined for it.
980
1015
  """
981
- if product_type not in self.product_types_config:
982
- raise NoMatchingProductType(product_type)
1016
+ if collection not in self.collections_config:
1017
+ raise NoMatchingCollection(collection)
983
1018
 
984
- return self.product_types_config[product_type].get("alias", product_type)
1019
+ if alias := self.collections_config[collection].alias:
1020
+ return alias
1021
+ return collection
985
1022
 
986
- def guess_product_type(
1023
+ def guess_collection(
987
1024
  self,
988
1025
  free_text: Optional[str] = None,
989
1026
  intersect: bool = False,
990
- instrument: Optional[str] = None,
1027
+ instruments: Optional[str] = None,
991
1028
  platform: Optional[str] = None,
992
- platformSerialIdentifier: Optional[str] = None,
993
- processingLevel: Optional[str] = None,
994
- sensorType: Optional[str] = None,
1029
+ constellation: Optional[str] = None,
1030
+ processing_level: Optional[str] = None,
1031
+ sensor_type: Optional[str] = None,
995
1032
  keywords: Optional[str] = None,
996
- abstract: Optional[str] = None,
1033
+ description: Optional[str] = None,
997
1034
  title: Optional[str] = None,
998
- missionStartDate: Optional[str] = None,
999
- missionEndDate: Optional[str] = None,
1035
+ start_date: Optional[str] = None,
1036
+ end_date: Optional[str] = None,
1000
1037
  **kwargs: Any,
1001
- ) -> list[str]:
1038
+ ) -> CollectionsList:
1002
1039
  """
1003
- Find EODAG product type IDs that best match a set of search parameters.
1040
+ Find EODAG collection IDs that best match a set of search parameters.
1004
1041
 
1005
- When using several filters, product types that match most of them will be returned at first.
1042
+ When using several filters, collections that match most of them will be returned at first.
1006
1043
 
1007
1044
  :param free_text: Free text search filter used to search accross all the following parameters. Handles logical
1008
1045
  operators with parenthesis (``AND``/``OR``/``NOT``), quoted phrases (``"exact phrase"``),
1009
1046
  ``*`` and ``?`` wildcards.
1010
1047
  :param intersect: Join results for each parameter using INTERSECT instead of UNION.
1011
- :param instrument: Instrument parameter.
1048
+ :param instruments: Instruments parameter.
1012
1049
  :param platform: Platform parameter.
1013
- :param platformSerialIdentifier: Platform serial identifier parameter.
1014
- :param processingLevel: Processing level parameter.
1015
- :param sensorType: Sensor type parameter.
1050
+ :param constellation: Constellation parameter.
1051
+ :param processing_level: Processing level parameter.
1052
+ :param sensor_type: Sensor type parameter.
1016
1053
  :param keywords: Keywords parameter.
1017
- :param abstract: Abstract parameter.
1054
+ :param description: description parameter.
1018
1055
  :param title: Title parameter.
1019
- :param missionStartDate: start date for datetime filtering. Not used by free_text
1020
- :param missionEndDate: end date for datetime filtering. Not used by free_text
1056
+ :param start_date: start date for datetime filtering. Not used by free_text
1057
+ :param end_date: end date for datetime filtering. Not used by free_text
1021
1058
  :returns: The best match for the given parameters.
1022
- :raises: :class:`~eodag.utils.exceptions.NoMatchingProductType`
1059
+ :raises: :class:`~eodag.utils.exceptions.NoMatchingCollection`
1023
1060
  """
1024
- if productType := kwargs.get("productType"):
1025
- return [productType]
1061
+ if collection := kwargs.get("collection"):
1062
+ try:
1063
+ collection = self.get_collection_from_alias(collection)
1064
+ return CollectionsList([self.collections_config[collection]])
1065
+ except NoMatchingCollection:
1066
+ return CollectionsList(
1067
+ [Collection.create_with_dag(self, id=collection)]
1068
+ )
1026
1069
 
1027
1070
  filters: dict[str, str] = {
1028
1071
  k: v
1029
1072
  for k, v in {
1030
- "instrument": instrument,
1073
+ "instruments": instruments,
1074
+ "constellation": constellation,
1031
1075
  "platform": platform,
1032
- "platformSerialIdentifier": platformSerialIdentifier,
1033
- "processingLevel": processingLevel,
1034
- "sensorType": sensorType,
1076
+ "processing:level": processing_level,
1077
+ "eodag:sensor_type": sensor_type,
1035
1078
  "keywords": keywords,
1036
- "abstract": abstract,
1079
+ "description": description,
1037
1080
  "title": title,
1038
1081
  }.items()
1039
1082
  if v is not None
@@ -1041,7 +1084,7 @@ class EODataAccessGateway:
1041
1084
 
1042
1085
  only_dates = (
1043
1086
  True
1044
- if (not free_text and not filters and (missionStartDate or missionEndDate))
1087
+ if (not free_text and not filters and (start_date or end_date))
1045
1088
  else False
1046
1089
  )
1047
1090
 
@@ -1051,11 +1094,10 @@ class EODataAccessGateway:
1051
1094
 
1052
1095
  guesses_with_score: list[tuple[str, int]] = []
1053
1096
 
1054
- for pt_id, pt_dict in self.product_types_config.source.items():
1097
+ for col, col_f in self.collections_config.items():
1055
1098
  if (
1056
- pt_id == GENERIC_PRODUCT_TYPE
1057
- or pt_id
1058
- not in self._plugins_manager.product_type_to_provider_config_map
1099
+ col == GENERIC_COLLECTION
1100
+ or col not in self._plugins_manager.collection_to_provider_config_map
1059
1101
  ):
1060
1102
  continue
1061
1103
 
@@ -1063,7 +1105,7 @@ class EODataAccessGateway:
1063
1105
 
1064
1106
  # free text search
1065
1107
  if free_text:
1066
- match = free_text_evaluator(pt_dict)
1108
+ match = free_text_evaluator(col_f.model_dump())
1067
1109
  if match:
1068
1110
  score += 1
1069
1111
  elif intersect:
@@ -1079,9 +1121,16 @@ class EODataAccessGateway:
1079
1121
  }
1080
1122
 
1081
1123
  filter_matches = [
1082
- filters_evaluators[filter_name]({filter_name: pt_dict[filter_name]})
1124
+ filters_evaluators[filter_name](
1125
+ {
1126
+ filter_name: col_f.__dict__[
1127
+ Collection.get_collection_mtd_from_alias(filter_name)
1128
+ ]
1129
+ }
1130
+ )
1083
1131
  for filter_name, value in filters.items()
1084
- if filter_name in pt_dict
1132
+ if Collection.get_collection_mtd_from_alias(filter_name)
1133
+ in col_f.__dict__
1085
1134
  ]
1086
1135
 
1087
1136
  if filters_matching_method(filter_matches):
@@ -1094,43 +1143,39 @@ class EODataAccessGateway:
1094
1143
  continue
1095
1144
 
1096
1145
  # datetime filtering
1097
- if missionStartDate or missionEndDate:
1146
+ if start_date or end_date:
1098
1147
  min_aware = datetime.datetime.min.replace(tzinfo=datetime.timezone.utc)
1099
1148
  max_aware = datetime.datetime.max.replace(tzinfo=datetime.timezone.utc)
1100
1149
 
1150
+ col_start = col_f.extent.temporal.interval[0][0]
1151
+ col_end = col_f.extent.temporal.interval[0][1]
1152
+
1101
1153
  max_start = max(
1102
- rfc3339_str_to_datetime(missionStartDate)
1103
- if missionStartDate
1104
- else min_aware,
1105
- rfc3339_str_to_datetime(pt_dict["missionStartDate"])
1106
- if pt_dict.get("missionStartDate")
1107
- else min_aware,
1154
+ rfc3339_str_to_datetime(start_date) if start_date else min_aware,
1155
+ col_start or min_aware,
1108
1156
  )
1109
1157
  min_end = min(
1110
- rfc3339_str_to_datetime(missionEndDate)
1111
- if missionEndDate
1112
- else max_aware,
1113
- rfc3339_str_to_datetime(pt_dict["missionEndDate"])
1114
- if pt_dict.get("missionEndDate")
1115
- else max_aware,
1158
+ rfc3339_str_to_datetime(end_date) if end_date else max_aware,
1159
+ col_end or max_aware,
1116
1160
  )
1117
1161
  if not (max_start <= min_end):
1118
1162
  continue
1119
1163
 
1120
- pt_alias = pt_dict.get("alias", pt_id)
1121
- guesses_with_score.append((pt_alias, score))
1164
+ guesses_with_score.append((col_f._id, score))
1122
1165
 
1123
1166
  if guesses_with_score:
1124
- # sort by score descending, then pt_id for stability
1167
+ # sort by score descending, then col for stability
1125
1168
  guesses_with_score.sort(key=lambda x: (-x[1], x[0]))
1126
- return [pt_id for pt_id, _ in guesses_with_score]
1169
+ return CollectionsList(
1170
+ [self.collections_config[col] for col, _ in guesses_with_score]
1171
+ )
1127
1172
 
1128
- raise NoMatchingProductType()
1173
+ raise NoMatchingCollection()
1129
1174
 
1130
1175
  def search(
1131
1176
  self,
1132
1177
  page: int = DEFAULT_PAGE,
1133
- items_per_page: int = DEFAULT_ITEMS_PER_PAGE,
1178
+ items_per_page: Optional[int] = DEFAULT_ITEMS_PER_PAGE,
1134
1179
  raise_errors: bool = False,
1135
1180
  start: Optional[str] = None,
1136
1181
  end: Optional[str] = None,
@@ -1138,20 +1183,22 @@ class EODataAccessGateway:
1138
1183
  locations: Optional[dict[str, str]] = None,
1139
1184
  provider: Optional[str] = None,
1140
1185
  count: bool = False,
1186
+ validate: Optional[bool] = True,
1141
1187
  **kwargs: Any,
1142
1188
  ) -> SearchResult:
1143
1189
  """Look for products matching criteria on known providers.
1144
1190
 
1145
1191
  The default behaviour is to look for products on the provider with the
1146
- highest priority supporting the requested product type. These priorities
1192
+ highest priority supporting the requested collection. These priorities
1147
1193
  are configurable through user configuration file or individual environment variable.
1148
1194
  If the request to the provider with the highest priority fails or is empty, the data
1149
1195
  will be request from the provider with the next highest priority.
1150
1196
  Only if the request fails for all available providers, an error will be thrown.
1151
1197
 
1152
- :param page: (optional) The page number to return
1198
+ :param page: (optional) The page number to return (**deprecated**, use
1199
+ :meth:`eodag.api.search_result.SearchResult.next_page` instead)
1153
1200
  :param items_per_page: (optional) The number of results that must appear in one single
1154
- page
1201
+ page. If ``None``, the maximum number possible will be used.
1155
1202
  :param raise_errors: (optional) When an error occurs when searching, if this is set to
1156
1203
  True, the error is raised
1157
1204
  :param start: (optional) Start sensing time in ISO 8601 format (e.g. "1990-11-26",
@@ -1178,9 +1225,11 @@ class EODataAccessGateway:
1178
1225
  If not set, the configured preferred provider will be used at first
1179
1226
  before trying others until finding results.
1180
1227
  :param count: (optional) Whether to run a query with a count request or not
1228
+ :param validate: (optional) Set to True to validate search parameters
1229
+ before sending the query to the provider
1181
1230
  :param kwargs: Some other criteria that will be used to do the search,
1182
1231
  using paramaters compatibles with the provider
1183
- :returns: A collection of EO products matching the criteria
1232
+ :returns: A set of EO products matching the criteria
1184
1233
 
1185
1234
  .. versionchanged:: v3.0.0b1
1186
1235
  ``search()`` method now returns only a single :class:`~eodag.api.search_result.SearchResult`
@@ -1191,6 +1240,15 @@ class EODataAccessGateway:
1191
1240
  return a list as a result of their processing. This requirement is
1192
1241
  enforced here.
1193
1242
  """
1243
+ if page != DEFAULT_PAGE:
1244
+ warnings.warn(
1245
+ "Usage of deprecated search parameter 'page' "
1246
+ "(Please use 'SearchResult.next_page()' instead)"
1247
+ " -- Deprecated since v3.9.0",
1248
+ DeprecationWarning,
1249
+ stacklevel=2,
1250
+ )
1251
+
1194
1252
  search_plugins, search_kwargs = self._prepare_search(
1195
1253
  start=start,
1196
1254
  end=end,
@@ -1200,28 +1258,38 @@ class EODataAccessGateway:
1200
1258
  **kwargs,
1201
1259
  )
1202
1260
  if search_kwargs.get("id"):
1261
+ # Don't validate requests by ID. "id" is not queryable.
1203
1262
  return self._search_by_id(
1204
1263
  search_kwargs.pop("id"),
1205
1264
  provider=provider,
1206
1265
  raise_errors=raise_errors,
1266
+ validate=False,
1207
1267
  **search_kwargs,
1208
1268
  )
1209
1269
  # remove datacube query string from kwargs which was only needed for search-by-id
1210
1270
  search_kwargs.pop("_dc_qs", None)
1211
-
1212
- search_kwargs.update(
1213
- page=page,
1214
- items_per_page=items_per_page,
1215
- )
1271
+ # add page parameter
1272
+ search_kwargs["page"] = page
1216
1273
 
1217
1274
  errors: list[tuple[str, Exception]] = []
1218
1275
  # Loop over available providers and return the first non-empty results
1219
1276
  for i, search_plugin in enumerate(search_plugins):
1220
1277
  search_plugin.clear()
1278
+
1279
+ # add appropriate items_per_page value
1280
+ search_kwargs["items_per_page"] = (
1281
+ items_per_page
1282
+ if items_per_page is not None
1283
+ else getattr(search_plugin.config, "pagination", {}).get(
1284
+ "max_items_per_page", DEFAULT_MAX_ITEMS_PER_PAGE
1285
+ )
1286
+ )
1287
+
1221
1288
  search_results = self._do_search(
1222
1289
  search_plugin,
1223
1290
  count=count,
1224
1291
  raise_errors=raise_errors,
1292
+ validate=validate,
1225
1293
  **search_kwargs,
1226
1294
  )
1227
1295
  errors.extend(search_results.errors)
@@ -1232,12 +1300,22 @@ class EODataAccessGateway:
1232
1300
  )
1233
1301
  elif len(search_results) > 0:
1234
1302
  search_results.errors = errors
1303
+ if count and search_results.number_matched:
1304
+ logger.info(
1305
+ "Found %s result(s) on provider '%s'",
1306
+ search_results.number_matched,
1307
+ search_results[0].provider,
1308
+ )
1235
1309
  return search_results
1236
1310
 
1237
1311
  if i > 1:
1238
1312
  logger.error("No result could be obtained from any available provider")
1239
1313
  return SearchResult([], 0, errors) if count else SearchResult([], errors=errors)
1240
1314
 
1315
+ @_deprecated(
1316
+ reason="Please use 'SearchResult.next_page()' instead",
1317
+ version="v3.9.0",
1318
+ )
1241
1319
  def search_iter_page(
1242
1320
  self,
1243
1321
  items_per_page: int = DEFAULT_ITEMS_PER_PAGE,
@@ -1249,6 +1327,9 @@ class EODataAccessGateway:
1249
1327
  ) -> Iterator[SearchResult]:
1250
1328
  """Iterate over the pages of a products search.
1251
1329
 
1330
+ .. deprecated:: v3.9.0
1331
+ Please use :meth:`eodag.api.search_result.SearchResult.next_page` instead.
1332
+
1252
1333
  :param items_per_page: (optional) The number of results requested per page
1253
1334
  :param start: (optional) Start sensing time in ISO 8601 format (e.g. "1990-11-26",
1254
1335
  "1990-11-26T14:30:10.153Z", "1990-11-26T14:30:10+02:00", ...).
@@ -1272,7 +1353,7 @@ class EODataAccessGateway:
1272
1353
  name=country and attr=ISO3
1273
1354
  :param kwargs: Some other criteria that will be used to do the search,
1274
1355
  using paramaters compatibles with the provider
1275
- :returns: An iterator that yields page per page a collection of EO products
1356
+ :returns: An iterator that yields page per page a set of EO products
1276
1357
  matching the criteria
1277
1358
  """
1278
1359
  search_plugins, search_kwargs = self._prepare_search(
@@ -1299,6 +1380,10 @@ class EODataAccessGateway:
1299
1380
  raise
1300
1381
  raise RequestError("No result could be obtained from any available provider")
1301
1382
 
1383
+ @_deprecated(
1384
+ reason="Please use 'SearchResult.next_page()' instead",
1385
+ version="v3.9.0",
1386
+ )
1302
1387
  def search_iter_page_plugin(
1303
1388
  self,
1304
1389
  search_plugin: Union[Search, Api],
@@ -1307,121 +1392,50 @@ class EODataAccessGateway:
1307
1392
  ) -> Iterator[SearchResult]:
1308
1393
  """Iterate over the pages of a products search using a given search plugin.
1309
1394
 
1395
+ .. deprecated:: v3.9.0
1396
+ Please use :meth:`eodag.api.search_result.SearchResult.next_page` instead.
1397
+
1310
1398
  :param items_per_page: (optional) The number of results requested per page
1311
1399
  :param kwargs: Some other criteria that will be used to do the search,
1312
1400
  using parameters compatibles with the provider
1313
1401
  :param search_plugin: search plugin to be used
1314
- :returns: An iterator that yields page per page a collection of EO products
1402
+ :returns: An iterator that yields page per page a set of EO products
1315
1403
  matching the criteria
1316
1404
  """
1317
-
1318
- iteration = 1
1319
- # Store the search plugin config pagination.next_page_url_tpl to reset it later
1320
- # since it might be modified if the next_page_url mechanism is used by the
1321
- # plugin. (same thing for next_page_query_obj, next_page_query_obj with POST reqs)
1322
- pagination_config = getattr(search_plugin.config, "pagination", {})
1323
- prev_next_page_url_tpl = pagination_config.get("next_page_url_tpl")
1324
- prev_next_page_query_obj = pagination_config.get("next_page_query_obj")
1325
- # Page has to be set to a value even if use_next is True, this is required
1326
- # internally by the search plugin (see collect_search_urls)
1327
1405
  kwargs.update(
1328
1406
  page=1,
1329
1407
  items_per_page=items_per_page,
1330
1408
  )
1331
- prev_product = None
1332
- next_page_url = None
1333
- next_page_query_obj = None
1334
- number_matched = None
1335
- while True:
1336
- # if count is enabled, it will only be performed on 1st iteration
1337
- if iteration == 2:
1338
- kwargs["count"] = False
1339
- if iteration > 1 and next_page_url:
1340
- pagination_config["next_page_url_tpl"] = next_page_url
1341
- if iteration > 1 and next_page_query_obj:
1342
- pagination_config["next_page_query_obj"] = next_page_query_obj
1343
- logger.info("Iterate search over multiple pages: page #%s", iteration)
1344
- try:
1345
- # remove unwanted kwargs for _do_search
1346
- kwargs.pop("raise_errors", None)
1347
- search_result = self._do_search(
1348
- search_plugin, raise_errors=True, **kwargs
1349
- )
1350
- # if count is enabled, it will only be performed on 1st iteration
1351
- if iteration == 1:
1352
- number_matched = search_result.number_matched
1353
- except Exception:
1354
- logger.warning(
1355
- "error at retrieval of data from %s, for params: %s",
1356
- search_plugin.provider,
1357
- str(kwargs),
1358
- )
1359
- raise
1360
- finally:
1361
- # we don't want that next(search_iter_page(...)) modifies the plugin
1362
- # indefinitely. So we reset after each request, but before the generator
1363
- # yields, the attr next_page_url (to None) and
1364
- # config.pagination["next_page_url_tpl"] (to its original value).
1365
- next_page_url = getattr(search_plugin, "next_page_url", None)
1366
- next_page_query_obj = getattr(search_plugin, "next_page_query_obj", {})
1367
- next_page_merge = getattr(search_plugin, "next_page_merge", None)
1368
-
1369
- if next_page_url:
1370
- search_plugin.next_page_url = None
1371
- if prev_next_page_url_tpl:
1372
- search_plugin.config.pagination[
1373
- "next_page_url_tpl"
1374
- ] = prev_next_page_url_tpl
1375
- if next_page_query_obj:
1376
- if prev_next_page_query_obj:
1377
- search_plugin.config.pagination[
1378
- "next_page_query_obj"
1379
- ] = prev_next_page_query_obj
1380
- # Update next_page_query_obj for next page req
1381
- if next_page_merge:
1382
- search_plugin.next_page_query_obj = dict(
1383
- getattr(search_plugin, "query_params", {}),
1384
- **next_page_query_obj,
1385
- )
1386
- else:
1387
- search_plugin.next_page_query_obj = next_page_query_obj
1388
-
1389
- if len(search_result) > 0:
1390
- # The first products between two iterations are compared. If they
1391
- # are actually the same product, it means the iteration failed at
1392
- # progressing for some reason. This is implemented as a workaround
1393
- # to some search plugins/providers not handling pagination.
1394
- product = search_result[0]
1395
- if (
1396
- prev_product
1397
- and product.properties["id"] == prev_product.properties["id"]
1398
- and product.provider == prev_product.provider
1399
- ):
1400
- logger.warning(
1401
- "Iterate over pages: stop iterating since the next page "
1402
- "appears to have the same products as in the previous one. "
1403
- "This provider may not implement pagination.",
1404
- )
1405
- last_page_with_products = iteration - 1
1406
- break
1407
- # use count got from 1st iteration
1408
- search_result.number_matched = number_matched
1409
- yield search_result
1410
- prev_product = product
1411
- # Prevent a last search if the current one returned less than the
1412
- # maximum number of items asked for.
1413
- if len(search_result) < items_per_page:
1414
- last_page_with_products = iteration
1415
- break
1416
- else:
1417
- last_page_with_products = iteration - 1
1409
+ try:
1410
+ # remove unwanted kwargs for _do_search
1411
+ kwargs.pop("raise_errors", None)
1412
+ search_result = self._do_search(search_plugin, raise_errors=True, **kwargs)
1413
+ search_result.raise_errors = True
1414
+
1415
+ except Exception:
1416
+ logger.warning(
1417
+ "error at retrieval of data from %s, for params: %s",
1418
+ search_plugin.provider,
1419
+ str(kwargs),
1420
+ )
1421
+ raise
1422
+
1423
+ if len(search_result) == 0:
1424
+ return
1425
+ # remove unwanted kwargs for next_page
1426
+ if kwargs.get("count") is True:
1427
+ kwargs["count"] = False
1428
+ kwargs.pop("page", None)
1429
+ search_result.search_params = kwargs
1430
+ if search_result._dag is None:
1431
+ search_result._dag = self
1432
+
1433
+ yield search_result
1434
+
1435
+ for next_result in search_result.next_page():
1436
+ if len(next_result) == 0:
1418
1437
  break
1419
- iteration += 1
1420
- kwargs["page"] = iteration
1421
- logger.debug(
1422
- "Iterate over pages: last products found on page %s",
1423
- last_page_with_products,
1424
- )
1438
+ yield next_result
1425
1439
 
1426
1440
  def search_all(
1427
1441
  self,
@@ -1471,84 +1485,45 @@ class EODataAccessGateway:
1471
1485
  name=country and attr=ISO3
1472
1486
  :param kwargs: Some other criteria that will be used to do the search,
1473
1487
  using parameters compatible with the provider
1474
- :returns: An iterator that yields page per page a collection of EO products
1488
+ :returns: An iterator that yields page per page a set of EO products
1475
1489
  matching the criteria
1476
1490
  """
1477
- # Get the search plugin and the maximized value
1478
- # of items_per_page if defined for the provider used.
1479
- try:
1480
- product_type = self.get_product_type_from_alias(
1481
- self.guess_product_type(**kwargs)[0]
1482
- )
1483
- except NoMatchingProductType:
1484
- product_type = GENERIC_PRODUCT_TYPE
1485
- else:
1486
- # fetch product types list if product_type is unknown
1487
- if (
1488
- product_type
1489
- not in self._plugins_manager.product_type_to_provider_config_map.keys()
1490
- ):
1491
- logger.debug(
1492
- f"Fetching external product types sources to find {product_type} product type"
1493
- )
1494
- self.fetch_product_types_list()
1495
-
1496
1491
  # remove unwanted count
1497
1492
  kwargs.pop("count", None)
1498
1493
 
1499
- search_plugins, search_kwargs = self._prepare_search(
1500
- start=start, end=end, geom=geom, locations=locations, **kwargs
1494
+ # First search
1495
+ search_results = self.search(
1496
+ items_per_page=items_per_page,
1497
+ start=start,
1498
+ end=end,
1499
+ geom=geom,
1500
+ locations=locations,
1501
+ **kwargs,
1501
1502
  )
1502
- for i, search_plugin in enumerate(search_plugins):
1503
- itp = (
1504
- items_per_page
1505
- or getattr(search_plugin.config, "pagination", {}).get(
1506
- "max_items_per_page"
1507
- )
1508
- or DEFAULT_MAX_ITEMS_PER_PAGE
1509
- )
1503
+ if len(search_results) == 0:
1504
+ return search_results
1505
+
1506
+ try:
1507
+ search_results.raise_errors = True
1508
+
1509
+ # consume iterator
1510
+ deque(search_results.next_page(update=True))
1511
+
1510
1512
  logger.info(
1511
- "Searching for all the products with provider %s and a maximum of %s "
1512
- "items per page.",
1513
- search_plugin.provider,
1514
- itp,
1513
+ "Found %s result(s) on provider '%s'",
1514
+ len(search_results),
1515
+ search_results[0].provider,
1516
+ )
1517
+ search_results.number_matched = len(search_results)
1518
+ except RequestError:
1519
+ logger.warning(
1520
+ "Found %s result(s) on provider '%s', but it may be incomplete "
1521
+ "as it ended with an error",
1522
+ len(search_results),
1523
+ search_results[0].provider,
1515
1524
  )
1516
- all_results = SearchResult([])
1517
- try:
1518
- for page_results in self.search_iter_page_plugin(
1519
- items_per_page=itp,
1520
- search_plugin=search_plugin,
1521
- count=False,
1522
- **search_kwargs,
1523
- ):
1524
- all_results.data.extend(page_results.data)
1525
- logger.info(
1526
- "Found %s result(s) on provider '%s'",
1527
- len(all_results),
1528
- search_plugin.provider,
1529
- )
1530
- return all_results
1531
- except RequestError:
1532
- if len(all_results) == 0 and i < len(search_plugins) - 1:
1533
- logger.warning(
1534
- "No result could be obtained from provider %s, "
1535
- "we will try to get the data from another provider",
1536
- search_plugin.provider,
1537
- )
1538
- elif len(all_results) == 0:
1539
- logger.error(
1540
- "No result could be obtained from any available provider"
1541
- )
1542
- raise
1543
- elif len(all_results) > 0:
1544
- logger.warning(
1545
- "Found %s result(s) on provider '%s', but it may be incomplete "
1546
- "as it ended with an error",
1547
- len(all_results),
1548
- search_plugin.provider,
1549
- )
1550
- return all_results
1551
- raise RequestError("No result could be obtained from any available provider")
1525
+
1526
+ return search_results
1552
1527
 
1553
1528
  def _search_by_id(
1554
1529
  self, uid: str, provider: Optional[str] = None, **kwargs: Any
@@ -1571,13 +1546,13 @@ class EODataAccessGateway:
1571
1546
  :param kwargs: Search criteria to help finding the right product
1572
1547
  :returns: A search result with one EO product or None at all
1573
1548
  """
1574
- product_type = kwargs.get("productType")
1575
- if product_type is not None:
1549
+ collection = kwargs.get("collection")
1550
+ if collection is not None:
1576
1551
  try:
1577
- product_type = self.get_product_type_from_alias(product_type)
1578
- except NoMatchingProductType:
1579
- logger.debug("product type %s not found", product_type)
1580
- get_search_plugins_kwargs = dict(provider=provider, product_type=product_type)
1552
+ collection = self.get_collection_from_alias(collection)
1553
+ except NoMatchingCollection:
1554
+ logger.debug("collection %s not found", collection)
1555
+ get_search_plugins_kwargs = dict(provider=provider, collection=collection)
1581
1556
  search_plugins = self._plugins_manager.get_search_plugins(
1582
1557
  **get_search_plugins_kwargs
1583
1558
  )
@@ -1632,10 +1607,10 @@ class EODataAccessGateway:
1632
1607
  results = filtered
1633
1608
 
1634
1609
  if len(results) == 1:
1635
- if not results[0].product_type:
1636
- # guess product type from properties
1637
- guesses = self.guess_product_type(**results[0].properties)
1638
- results[0].product_type = guesses[0]
1610
+ if not results[0].collection:
1611
+ # guess collection from properties
1612
+ guesses = self.guess_collection(**results[0].properties)
1613
+ results[0].collection = guesses[0].id
1639
1614
  # reset driver
1640
1615
  results[0].driver = results[0].get_driver()
1641
1616
  results.number_matched = 1
@@ -1647,15 +1622,15 @@ class EODataAccessGateway:
1647
1622
  )
1648
1623
  return SearchResult([], 0, results.errors)
1649
1624
 
1650
- def _fetch_external_product_type(self, provider: str, product_type: str):
1625
+ def _fetch_external_collection(self, provider: str, collection: str):
1651
1626
  plugins = self._plugins_manager.get_search_plugins(provider=provider)
1652
1627
  plugin = next(plugins)
1653
1628
 
1654
1629
  # check after plugin init if still fetchable
1655
- if not getattr(plugin.config, "discover_product_types", {}).get("fetch_url"):
1630
+ if not getattr(plugin.config, "discover_collections", {}).get("fetch_url"):
1656
1631
  return None
1657
1632
 
1658
- kwargs: dict[str, Any] = {"productType": product_type}
1633
+ kwargs: dict[str, Any] = {"collection": collection}
1659
1634
 
1660
1635
  # append auth if needed
1661
1636
  if getattr(plugin.config, "need_auth", False):
@@ -1666,8 +1641,8 @@ class EODataAccessGateway:
1666
1641
  ):
1667
1642
  kwargs["auth"] = auth
1668
1643
 
1669
- product_type_config = plugin.discover_product_types(**kwargs)
1670
- self.update_product_types_list({provider: product_type_config})
1644
+ collection_config = plugin.discover_collections(**kwargs)
1645
+ self.update_collections_list({provider: collection_config})
1671
1646
 
1672
1647
  def _prepare_search(
1673
1648
  self,
@@ -1683,9 +1658,9 @@ class EODataAccessGateway:
1683
1658
  Product query:
1684
1659
  * By id (plus optional 'provider')
1685
1660
  * By search params:
1686
- * productType query:
1687
- * By product type (e.g. 'S2_MSI_L1C')
1688
- * By params (e.g. 'platform'), see guess_product_type
1661
+ * collection query:
1662
+ * By collection (e.g. 'S2_MSI_L1C')
1663
+ * By params (e.g. 'platform'), see guess_collection
1689
1664
  * dates: 'start' and/or 'end'
1690
1665
  * geometry: 'geom' or 'bbox' or 'box'
1691
1666
  * search locations
@@ -1700,53 +1675,53 @@ class EODataAccessGateway:
1700
1675
  If no time offset is given, the time is assumed to be given in UTC.
1701
1676
  :param geom: (optional) Search area that can be defined in different ways (see search)
1702
1677
  :param locations: (optional) Location filtering by name using locations configuration
1703
- :param provider: provider to be used, if no provider is given or the product type
1678
+ :param provider: provider to be used, if no provider is given or the collection
1704
1679
  is not available for the provider, the preferred provider is used
1705
1680
  :param kwargs: Some other criteria
1706
1681
  * id and/or a provider for a search by
1707
- * search criteria to guess the product type
1682
+ * search criteria to guess the collection
1708
1683
  * other criteria compatible with the provider
1709
1684
  :returns: Search plugins list and the prepared kwargs to make a query.
1710
1685
  """
1711
- product_type: Optional[str] = kwargs.get("productType")
1712
- if product_type is None:
1686
+ collection: Optional[str] = kwargs.get("collection")
1687
+ if collection is None:
1713
1688
  try:
1714
- guesses = self.guess_product_type(**kwargs)
1689
+ guesses = self.guess_collection(**kwargs)
1715
1690
 
1716
- # guess_product_type raises a NoMatchingProductType error if no product
1691
+ # guess_collection raises a NoMatchingCollection error if no product
1717
1692
  # is found. Here, the supported search params are removed from the
1718
1693
  # kwargs if present, not to propagate them to the query itself.
1719
1694
  for param in (
1720
- "instrument",
1695
+ "instruments",
1696
+ "constellation",
1721
1697
  "platform",
1722
- "platformSerialIdentifier",
1723
- "processingLevel",
1724
- "sensorType",
1698
+ "processing:level",
1699
+ "eodag:sensor_type",
1725
1700
  ):
1726
1701
  kwargs.pop(param, None)
1727
1702
 
1728
1703
  # By now, only use the best bet
1729
- product_type = guesses[0]
1730
- except NoMatchingProductType:
1704
+ collection = guesses[0].id
1705
+ except NoMatchingCollection:
1731
1706
  queried_id = kwargs.get("id")
1732
1707
  if queried_id is None:
1733
1708
  logger.info(
1734
- "No product type could be guessed with provided arguments"
1709
+ "No collection could be guessed with provided arguments"
1735
1710
  )
1736
1711
  else:
1737
1712
  return [], kwargs
1738
1713
 
1739
- if product_type is not None:
1714
+ if collection is not None:
1740
1715
  try:
1741
- product_type = self.get_product_type_from_alias(product_type)
1742
- except NoMatchingProductType:
1743
- logger.info("unknown product type " + product_type)
1744
- kwargs["productType"] = product_type
1716
+ collection = self.get_collection_from_alias(collection)
1717
+ except NoMatchingCollection:
1718
+ logger.info("unknown collection " + collection)
1719
+ kwargs["collection"] = collection
1745
1720
 
1746
1721
  if start is not None:
1747
- kwargs["startTimeFromAscendingNode"] = start
1722
+ kwargs["start_datetime"] = start
1748
1723
  if end is not None:
1749
- kwargs["completionTimeFromAscendingNode"] = end
1724
+ kwargs["end_datetime"] = end
1750
1725
  if "box" in kwargs or "bbox" in kwargs:
1751
1726
  logger.warning(
1752
1727
  "'box' or 'bbox' parameters are only supported for backwards "
@@ -1767,33 +1742,44 @@ class EODataAccessGateway:
1767
1742
  kwargs.pop(arg, None)
1768
1743
  del kwargs["locations"]
1769
1744
 
1770
- # fetch product types list if product_type is unknown
1745
+ # fetch collections list if collection is unknown
1771
1746
  if (
1772
- product_type
1773
- not in self._plugins_manager.product_type_to_provider_config_map.keys()
1747
+ collection
1748
+ not in self._plugins_manager.collection_to_provider_config_map.keys()
1774
1749
  ):
1775
- if provider and product_type:
1776
- # Try to get specific product type from external provider
1777
- logger.debug(f"Fetching {provider} to find {product_type} product type")
1778
- self._fetch_external_product_type(provider, product_type)
1750
+ if provider and collection:
1751
+ # fetch ref for given provider and collection
1752
+ logger.debug(
1753
+ f"Fetching external collections sources to find {provider} {collection} collection"
1754
+ )
1755
+ self.fetch_collections_list(provider)
1756
+ if (
1757
+ collection
1758
+ not in self._plugins_manager.collection_to_provider_config_map.keys()
1759
+ ):
1760
+ # Try to get specific collection from external provider
1761
+ logger.debug(
1762
+ "Fetching %s to find %s collection", provider, collection
1763
+ )
1764
+ self._fetch_external_collection(provider, collection)
1779
1765
  if not provider:
1780
- # no provider or still not found -> fetch all external product types
1766
+ # no provider or still not found -> fetch all external collections
1781
1767
  logger.debug(
1782
- f"Fetching external product types sources to find {product_type} product type"
1768
+ f"Fetching external collections sources to find {collection} collection"
1783
1769
  )
1784
- self.fetch_product_types_list()
1770
+ self.fetch_collections_list()
1785
1771
 
1786
1772
  preferred_provider = self.get_preferred_provider()[0]
1787
1773
 
1788
1774
  search_plugins: list[Union[Search, Api]] = []
1789
1775
  for plugin in self._plugins_manager.get_search_plugins(
1790
- product_type=product_type, provider=provider
1776
+ collection=collection, provider=provider
1791
1777
  ):
1792
- # exclude MeteoblueSearch plugins from search fallback for unknown product_type
1778
+ # exclude MeteoblueSearch plugins from search fallback for unknown collection
1793
1779
  if (
1794
1780
  provider != plugin.provider
1795
1781
  and preferred_provider != plugin.provider
1796
- and product_type not in self.product_types_config
1782
+ and collection not in self.collections_config
1797
1783
  and isinstance(plugin, MeteoblueSearch)
1798
1784
  ):
1799
1785
  continue
@@ -1804,8 +1790,8 @@ class EODataAccessGateway:
1804
1790
  providers = [plugin.provider for plugin in search_plugins]
1805
1791
  if provider not in providers:
1806
1792
  logger.debug(
1807
- "Product type '%s' is not available with preferred provider '%s'.",
1808
- product_type,
1793
+ "Collection '%s' is not available with preferred provider '%s'.",
1794
+ collection,
1809
1795
  provider,
1810
1796
  )
1811
1797
  else:
@@ -1814,11 +1800,11 @@ class EODataAccessGateway:
1814
1800
  )[0]
1815
1801
  search_plugins.remove(provider_plugin)
1816
1802
  search_plugins.insert(0, provider_plugin)
1817
- # Add product_types_config to plugin config. This dict contains product
1803
+ # Add collections_config to plugin config. This dict contains product
1818
1804
  # type metadata that will also be stored in each product's properties.
1819
1805
  for search_plugin in search_plugins:
1820
- if product_type is not None:
1821
- self._attach_product_type_config(search_plugin, product_type)
1806
+ if collection is not None:
1807
+ self._attach_collection_config(search_plugin, collection)
1822
1808
 
1823
1809
  return search_plugins, kwargs
1824
1810
 
@@ -1827,6 +1813,7 @@ class EODataAccessGateway:
1827
1813
  search_plugin: Union[Search, Api],
1828
1814
  count: bool = False,
1829
1815
  raise_errors: bool = False,
1816
+ validate: Optional[bool] = True,
1830
1817
  **kwargs: Any,
1831
1818
  ) -> SearchResult:
1832
1819
  """Internal method that performs a search on a given provider.
@@ -1836,6 +1823,8 @@ class EODataAccessGateway:
1836
1823
  :param raise_errors: (optional) When an error occurs when searching, if this is set to
1837
1824
  True, the error is raised
1838
1825
  :param kwargs: Some other criteria that will be used to do the search
1826
+ :param validate: (optional) Set to True to validate search parameters
1827
+ before sending the query to the provider
1839
1828
  :returns: A collection of EO products matching the criteria
1840
1829
  """
1841
1830
  logger.info("Searching on provider %s", search_plugin.provider)
@@ -1856,13 +1845,11 @@ class EODataAccessGateway:
1856
1845
  max_items_per_page,
1857
1846
  )
1858
1847
 
1859
- results: list[EOProduct] = []
1860
- total_results: Optional[int] = 0 if count else None
1861
-
1862
1848
  errors: list[tuple[str, Exception]] = []
1863
1849
 
1864
1850
  try:
1865
1851
  prep = PreparedSearch(count=count)
1852
+ prep.raise_errors = raise_errors
1866
1853
 
1867
1854
  # append auth if needed
1868
1855
  if getattr(search_plugin.config, "need_auth", False):
@@ -1873,17 +1860,57 @@ class EODataAccessGateway:
1873
1860
  ):
1874
1861
  prep.auth = auth
1875
1862
 
1876
- prep.page = kwargs.pop("page", None)
1877
1863
  prep.items_per_page = kwargs.pop("items_per_page", None)
1864
+ prep.next_page_token = kwargs.pop("next_page_token", None)
1865
+ prep.next_page_token_key = kwargs.pop(
1866
+ "next_page_token_key", None
1867
+ ) or search_plugin.config.pagination.get("next_page_token_key", "page")
1868
+ prep.page = kwargs.pop("page", None)
1878
1869
 
1879
- res, nb_res = search_plugin.query(prep, **kwargs)
1870
+ if (
1871
+ prep.next_page_token_key == "page"
1872
+ and prep.items_per_page is not None
1873
+ and prep.next_page_token is None
1874
+ and prep.page is not None
1875
+ ):
1876
+ prep.next_page_token = str(
1877
+ prep.page
1878
+ - 1
1879
+ + search_plugin.config.pagination.get("start_page", DEFAULT_PAGE)
1880
+ )
1881
+
1882
+ # remove None values and convert param names to their pydantic alias if any
1883
+ search_params = {}
1884
+ ecmwf_queryables = [
1885
+ f"{ECMWF_PREFIX[:-1]}_{k}" for k in ECMWF_ALLOWED_KEYWORDS
1886
+ ]
1887
+ for param, value in kwargs.items():
1888
+ if value is None:
1889
+ continue
1890
+ if param in Queryables.model_fields:
1891
+ param_alias = Queryables.model_fields[param].alias or param
1892
+ search_params[param_alias] = value
1893
+ elif param in ecmwf_queryables:
1894
+ # alias equivalent for ECMWF queryables
1895
+ search_params[
1896
+ re.sub(rf"^{ECMWF_PREFIX[:-1]}_", f"{ECMWF_PREFIX}", param)
1897
+ ] = value
1898
+ else:
1899
+ # remove `provider:` or `provider_` prefix if any
1900
+ search_params[
1901
+ re.sub(r"^" + search_plugin.provider + r"[_:]", "", param)
1902
+ ] = value
1903
+
1904
+ if validate:
1905
+ search_plugin.validate(search_params, prep.auth)
1880
1906
 
1881
- if not isinstance(res, list):
1907
+ search_result = search_plugin.query(prep, **search_params)
1908
+
1909
+ if not isinstance(search_result.data, list):
1882
1910
  raise PluginImplementationError(
1883
1911
  "The query function of a Search plugin must return a list of "
1884
- "results, got {} instead".format(type(res))
1912
+ "results, got {} instead".format(type(search_result.data))
1885
1913
  )
1886
-
1887
1914
  # Filter and attach to each eoproduct in the result the plugin capable of
1888
1915
  # downloading it (this is done to enable the eo_product to download itself
1889
1916
  # doing: eo_product.download()). The filtering is done by keeping only
@@ -1893,56 +1920,51 @@ class EODataAccessGateway:
1893
1920
  # WARNING: this means an eo_product that has an invalid geometry can still
1894
1921
  # be returned as a search result if there was no search extent (because we
1895
1922
  # will not try to do an intersection)
1896
- for eo_product in res:
1897
- # if product_type is not defined, try to guess using properties
1898
- if eo_product.product_type is None:
1923
+ for eo_product in search_result.data:
1924
+ # if collection is not defined, try to guess using properties
1925
+ if eo_product.collection is None:
1899
1926
  pattern = re.compile(r"[^\w,]+")
1900
1927
  try:
1901
- guesses = self.guess_product_type(
1928
+ guesses = self.guess_collection(
1902
1929
  intersect=False,
1903
1930
  **{
1904
1931
  k: pattern.sub("", str(v).upper())
1905
1932
  for k, v in eo_product.properties.items()
1906
1933
  if k
1907
1934
  in [
1908
- "instrument",
1935
+ "instruments",
1936
+ "constellation",
1909
1937
  "platform",
1910
- "platformSerialIdentifier",
1911
- "processingLevel",
1912
- "sensorType",
1938
+ "processing:level",
1939
+ "eodag:sensor_type",
1913
1940
  "keywords",
1914
1941
  ]
1915
1942
  and v is not None
1916
1943
  },
1917
1944
  )
1918
- except NoMatchingProductType:
1945
+ except NoMatchingCollection:
1919
1946
  pass
1920
1947
  else:
1921
- eo_product.product_type = guesses[0]
1948
+ eo_product.collection = guesses[0].id
1922
1949
 
1923
1950
  try:
1924
- if eo_product.product_type is not None:
1925
- eo_product.product_type = self.get_product_type_from_alias(
1926
- eo_product.product_type
1951
+ if eo_product.collection is not None:
1952
+ eo_product.collection = self.get_collection_from_alias(
1953
+ eo_product.collection
1927
1954
  )
1928
- except NoMatchingProductType:
1929
- logger.debug("product type %s not found", eo_product.product_type)
1955
+ except NoMatchingCollection:
1956
+ logger.debug("collection %s not found", eo_product.collection)
1930
1957
 
1931
1958
  if eo_product.search_intersection is not None:
1932
1959
  eo_product._register_downloader_from_manager(self._plugins_manager)
1933
1960
 
1934
- results.extend(res)
1935
- total_results = (
1936
- None
1937
- if (nb_res is None or total_results is None)
1938
- else total_results + nb_res
1939
- )
1940
- if count and nb_res is not None:
1941
- logger.info(
1942
- "Found %s result(s) on provider '%s'",
1943
- nb_res,
1944
- search_plugin.provider,
1945
- )
1961
+ # Make next_page not available if the current one returned less than the maximum number of items asked for.
1962
+ if not prep.items_per_page or len(search_result) < prep.items_per_page:
1963
+ search_result.next_page_token = None
1964
+
1965
+ search_result._dag = self
1966
+ return search_result
1967
+
1946
1968
  except Exception as e:
1947
1969
  if raise_errors:
1948
1970
  # Raise the error, letting the application wrapping eodag know that
@@ -1954,7 +1976,7 @@ class EODataAccessGateway:
1954
1976
  search_plugin.provider,
1955
1977
  )
1956
1978
  errors.append((search_plugin.provider, e))
1957
- return SearchResult(results, total_results, errors)
1979
+ return SearchResult([], 0, errors)
1958
1980
 
1959
1981
  def crunch(self, results: SearchResult, **kwargs: Any) -> SearchResult:
1960
1982
  """Apply the filters given through the keyword arguments to the results
@@ -2005,7 +2027,7 @@ class EODataAccessGateway:
2005
2027
  ) -> list[str]:
2006
2028
  """Download all products resulting from a search.
2007
2029
 
2008
- :param search_result: A collection of EO products resulting from a search
2030
+ :param search_result: A set of EO products resulting from a search
2009
2031
  :param downloaded_callback: (optional) A method or a callable object which takes
2010
2032
  as parameter the ``product``. You can use the base class
2011
2033
  :class:`~eodag.utils.DownloadedCallback` and override
@@ -2058,13 +2080,16 @@ class EODataAccessGateway:
2058
2080
  search_result: SearchResult, filename: str = "search_results.geojson"
2059
2081
  ) -> str:
2060
2082
  """Registers results of a search into a geojson file.
2083
+ The output is a FeatureCollection containing the EO products as features,
2084
+ with additional metadata such as ``number_matched``, ``next_page_token``,
2085
+ and ``search_params`` stored in the properties.
2061
2086
 
2062
- :param search_result: A collection of EO products resulting from a search
2087
+ :param search_result: A set of EO products resulting from a search
2063
2088
  :param filename: (optional) The name of the file to generate
2064
2089
  :returns: The name of the created file
2065
2090
  """
2066
2091
  with open(filename, "w") as fh:
2067
- geojson.dump(search_result, fh)
2092
+ geojson.dump(search_result.as_geojson_object(), fh)
2068
2093
  return filename
2069
2094
 
2070
2095
  @staticmethod
@@ -2079,12 +2104,16 @@ class EODataAccessGateway:
2079
2104
 
2080
2105
  def deserialize_and_register(self, filename: str) -> SearchResult:
2081
2106
  """Loads results of a search from a geojson file and register
2082
- products with the information needed to download itself
2107
+ products with the information needed to download itself.
2108
+
2109
+ This method also sets the internal EODataAccessGateway instance on the products,
2110
+ enabling pagination (e.g. access to next pages) if available.
2083
2111
 
2084
2112
  :param filename: A filename containing a search result encoded as a geojson
2085
- :returns: The search results encoded in `filename`
2113
+ :returns: The search results encoded in `filename`, ready for download and pagination
2086
2114
  """
2087
2115
  products = self.deserialize(filename)
2116
+ products._dag = self
2088
2117
  for i, product in enumerate(products):
2089
2118
  if product.downloader is None:
2090
2119
  downloader = self._plugins_manager.get_download_plugin(product)
@@ -2095,69 +2124,6 @@ class EODataAccessGateway:
2095
2124
 
2096
2125
  return products
2097
2126
 
2098
- @_deprecated(
2099
- reason="Use the StaticStacSearch search plugin instead", version="2.2.1"
2100
- )
2101
- def load_stac_items(
2102
- self,
2103
- filename: str,
2104
- recursive: bool = False,
2105
- max_connections: int = 100,
2106
- provider: Optional[str] = None,
2107
- productType: Optional[str] = None,
2108
- timeout: int = HTTP_REQ_TIMEOUT,
2109
- ssl_verify: bool = True,
2110
- **kwargs: Any,
2111
- ) -> SearchResult:
2112
- """Loads STAC items from a geojson file / STAC catalog or collection, and convert to SearchResult.
2113
-
2114
- Features are parsed using eodag provider configuration, as if they were
2115
- the response content to an API request.
2116
-
2117
- :param filename: A filename containing features encoded as a geojson
2118
- :param recursive: (optional) Browse recursively in child nodes if True
2119
- :param max_connections: (optional) Maximum number of connections for concurrent HTTP requests
2120
- :param provider: (optional) Data provider
2121
- :param productType: (optional) Data product type
2122
- :param timeout: (optional) Timeout in seconds for each internal HTTP request
2123
- :param kwargs: Parameters that will be stored in the result as
2124
- search criteria
2125
- :returns: The search results encoded in `filename`
2126
-
2127
- .. deprecated:: 2.2.1
2128
- Use the :class:`~eodag.plugins.search.static_stac_search.StaticStacSearch` search plugin instead.
2129
- """
2130
- features = fetch_stac_items(
2131
- filename,
2132
- recursive=recursive,
2133
- max_connections=max_connections,
2134
- timeout=timeout,
2135
- ssl_verify=ssl_verify,
2136
- )
2137
- feature_collection = geojson.FeatureCollection(features)
2138
-
2139
- plugin = next(
2140
- self._plugins_manager.get_search_plugins(
2141
- product_type=productType, provider=provider
2142
- )
2143
- )
2144
- # save plugin._request and mock it to make return loaded static results
2145
- plugin_request = plugin._request
2146
- plugin._request = (
2147
- lambda url, info_message=None, exception_message=None: MockResponse(
2148
- feature_collection, 200
2149
- )
2150
- )
2151
-
2152
- search_result = self.search(
2153
- productType=productType, provider=provider, **kwargs
2154
- )
2155
-
2156
- # restore plugin._request
2157
- plugin._request = plugin_request
2158
-
2159
- return search_result
2160
-
2161
2127
  def download(
2162
2128
  self,
2163
2129
  product: EOProduct,
@@ -2173,16 +2139,16 @@ class EODataAccessGateway:
2173
2139
  checks like verifying that a downloader and authenticator are registered
2174
2140
  for the product before trying to download it.
2175
2141
 
2176
- If the metadata mapping for ``downloadLink`` is set to something that can be
2142
+ If the metadata mapping for ``eodag:download_link`` is set to something that can be
2177
2143
  interpreted as a link on a
2178
2144
  local filesystem, the download is skipped (by now, only a link starting
2179
2145
  with ``file:/`` is supported). Therefore, any user that knows how to extract
2180
2146
  product location from product metadata on a provider can override the
2181
- ``downloadLink`` metadata mapping in the right way. For example, using the
2147
+ ``eodag:download_link`` metadata mapping in the right way. For example, using the
2182
2148
  environment variable:
2183
- ``EODAG__CREODIAS__SEARCH__METADATA_MAPPING__DOWNLOADLINK="file:///{id}"`` will
2149
+ ``EODAG__CREODIAS__SEARCH__METADATA_MAPPING__EODAG_DOWNLOAD_LINK="file:///{id}"`` will
2184
2150
  lead to all :class:`~eodag.api.product._product.EOProduct`'s originating from the
2185
- provider ``creodias`` to have their ``downloadLink`` metadata point to something like:
2151
+ provider ``creodias`` to have their ``eodag:download_link`` metadata point to something like:
2186
2152
  ``file:///12345-678``, making this method immediately return the later string without
2187
2153
  trying to download the product.
2188
2154
 
@@ -2246,46 +2212,46 @@ class EODataAccessGateway:
2246
2212
  fetch_providers: bool = True,
2247
2213
  **kwargs: Any,
2248
2214
  ) -> QueryablesDict:
2249
- """Fetch the queryable properties for a given product type and/or provider.
2215
+ """Fetch the queryable properties for a given collection and/or provider.
2250
2216
 
2251
2217
  :param provider: (optional) The provider.
2252
- :param fetch_providers: If new product types should be fetched from the providers; default: True
2253
- :param kwargs: additional filters for queryables (`productType` or other search
2218
+ :param fetch_providers: If new collections should be fetched from the providers; default: True
2219
+ :param kwargs: additional filters for queryables (`collection` or other search
2254
2220
  arguments)
2255
2221
 
2256
- :raises UnsupportedProductType: If the specified product type is not available for the
2222
+ :raises UnsupportedCollection: If the specified collection is not available for the
2257
2223
  provider.
2258
2224
 
2259
2225
  :returns: A :class:`~eodag.api.product.queryables.QuerybalesDict` containing the EODAG queryable
2260
2226
  properties, associating parameters to their annotated type, and a additional_properties attribute
2261
2227
  """
2262
- # only fetch providers if product type is not found
2263
- available_product_types: list[str] = [
2264
- pt["ID"]
2265
- for pt in self.list_product_types(provider=provider, fetch_providers=False)
2228
+ # only fetch providers if collection is not found
2229
+ available_collections: list[str] = [
2230
+ col.id
2231
+ for col in self.list_collections(provider=provider, fetch_providers=False)
2266
2232
  ]
2267
- product_type: Optional[str] = kwargs.get("productType")
2268
- pt_alias: Optional[str] = product_type
2233
+ collection: Optional[str] = kwargs.get("collection")
2234
+ coll_alias: Optional[str] = collection
2269
2235
 
2270
- if product_type:
2271
- if product_type not in available_product_types:
2236
+ if collection:
2237
+ if collection not in available_collections:
2272
2238
  if fetch_providers:
2273
2239
  # fetch providers and try again
2274
- available_product_types = [
2275
- pt["ID"]
2276
- for pt in self.list_product_types(
2240
+ available_collections = [
2241
+ col.id
2242
+ for col in self.list_collections(
2277
2243
  provider=provider, fetch_providers=True
2278
2244
  )
2279
2245
  ]
2280
- raise UnsupportedProductType(f"{product_type} is not available.")
2246
+ raise UnsupportedCollection(f"{collection} is not available.")
2281
2247
  try:
2282
- kwargs["productType"] = product_type = self.get_product_type_from_alias(
2283
- product_type
2248
+ kwargs["collection"] = collection = self.get_collection_from_alias(
2249
+ collection
2284
2250
  )
2285
- except NoMatchingProductType as e:
2286
- raise UnsupportedProductType(f"{product_type} is not available.") from e
2251
+ except NoMatchingCollection as e:
2252
+ raise UnsupportedCollection(f"{collection} is not available.") from e
2287
2253
 
2288
- if not provider and not product_type:
2254
+ if not provider and not collection:
2289
2255
  return QueryablesDict(
2290
2256
  additional_properties=True,
2291
2257
  **model_fields_to_annotated(CommonQueryables.model_fields),
@@ -2295,16 +2261,16 @@ class EODataAccessGateway:
2295
2261
  additional_information = []
2296
2262
  queryable_properties: dict[str, Any] = {}
2297
2263
 
2298
- for plugin in self._plugins_manager.get_search_plugins(product_type, provider):
2299
- # attach product type config
2300
- product_type_configs: dict[str, Any] = {}
2301
- if product_type:
2302
- self._attach_product_type_config(plugin, product_type)
2303
- product_type_configs[product_type] = plugin.config.product_type_config
2264
+ for plugin in self._plugins_manager.get_search_plugins(collection, provider):
2265
+ # attach collection config
2266
+ collection_configs: dict[str, Any] = {}
2267
+ if collection:
2268
+ self._attach_collection_config(plugin, collection)
2269
+ collection_configs[collection] = plugin.config.collection_config
2304
2270
  else:
2305
- for pt in available_product_types:
2306
- self._attach_product_type_config(plugin, pt)
2307
- product_type_configs[pt] = plugin.config.product_type_config
2271
+ for col in available_collections:
2272
+ self._attach_collection_config(plugin, col)
2273
+ collection_configs[col] = plugin.config.collection_config
2308
2274
 
2309
2275
  # authenticate if required
2310
2276
  if getattr(plugin.config, "need_auth", False) and (
@@ -2326,10 +2292,10 @@ class EODataAccessGateway:
2326
2292
 
2327
2293
  plugin_queryables = plugin.list_queryables(
2328
2294
  kwargs_alias,
2329
- available_product_types,
2330
- product_type_configs,
2331
- product_type,
2332
- pt_alias,
2295
+ available_collections,
2296
+ collection_configs,
2297
+ collection,
2298
+ coll_alias,
2333
2299
  )
2334
2300
 
2335
2301
  if plugin_queryables.additional_information:
@@ -2379,32 +2345,31 @@ class EODataAccessGateway:
2379
2345
  }
2380
2346
  return sortables
2381
2347
 
2382
- def _attach_product_type_config(self, plugin: Search, product_type: str) -> None:
2348
+ def _attach_collection_config(self, plugin: Search, collection: str) -> None:
2383
2349
  """
2384
- Attach product_types_config to plugin config. This dict contains product
2350
+ Attach collections_config to plugin config. This dict contains product
2385
2351
  type metadata that will also be stored in each product's properties.
2386
2352
  """
2387
2353
  try:
2388
- plugin.config.product_type_config = dict(
2354
+ plugin.config.collection_config = dict(
2389
2355
  [
2390
- p
2391
- for p in self.list_product_types(
2356
+ c.model_dump(mode="json", exclude={"id"})
2357
+ for c in self.list_collections(
2392
2358
  plugin.provider, fetch_providers=False
2393
2359
  )
2394
- if p["_id"] == product_type
2360
+ if c._id == collection
2395
2361
  ][0],
2396
- **{"productType": product_type},
2362
+ **{"collection": collection},
2397
2363
  )
2398
- # If the product isn't in the catalog, it's a generic product type.
2364
+ # If the product isn't in the catalog, it's a generic collection.
2399
2365
  except IndexError:
2400
- # Construct the GENERIC_PRODUCT_TYPE metadata
2401
- plugin.config.product_type_config = dict(
2402
- ID=GENERIC_PRODUCT_TYPE,
2403
- **self.product_types_config[GENERIC_PRODUCT_TYPE],
2404
- productType=product_type,
2366
+ # Construct the GENERIC_COLLECTION metadata
2367
+ plugin.config.collection_config = dict(
2368
+ **self.collections_config[GENERIC_COLLECTION].model_dump(
2369
+ mode="json", exclude={"id"}
2370
+ ),
2371
+ collection=collection,
2405
2372
  )
2406
- # Remove the ID since this is equal to productType.
2407
- plugin.config.product_type_config.pop("ID", None)
2408
2373
 
2409
2374
  def import_stac_items(self, items_urls: list[str]) -> SearchResult:
2410
2375
  """Import STAC items from a list of URLs and convert them to SearchResult.