eodag 3.10.1__py3-none-any.whl → 4.0.0a2__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (75) hide show
  1. eodag/__init__.py +6 -1
  2. eodag/api/collection.py +353 -0
  3. eodag/api/core.py +606 -641
  4. eodag/api/product/__init__.py +3 -3
  5. eodag/api/product/_product.py +74 -56
  6. eodag/api/product/drivers/__init__.py +4 -46
  7. eodag/api/product/drivers/base.py +0 -28
  8. eodag/api/product/metadata_mapping.py +178 -216
  9. eodag/api/search_result.py +156 -15
  10. eodag/cli.py +83 -403
  11. eodag/config.py +81 -51
  12. eodag/plugins/apis/base.py +2 -2
  13. eodag/plugins/apis/ecmwf.py +36 -25
  14. eodag/plugins/apis/usgs.py +55 -40
  15. eodag/plugins/authentication/base.py +1 -3
  16. eodag/plugins/crunch/filter_date.py +3 -3
  17. eodag/plugins/crunch/filter_latest_intersect.py +2 -2
  18. eodag/plugins/crunch/filter_latest_tpl_name.py +1 -1
  19. eodag/plugins/download/aws.py +46 -42
  20. eodag/plugins/download/base.py +13 -14
  21. eodag/plugins/download/http.py +65 -65
  22. eodag/plugins/manager.py +28 -29
  23. eodag/plugins/search/__init__.py +6 -4
  24. eodag/plugins/search/base.py +131 -80
  25. eodag/plugins/search/build_search_result.py +245 -173
  26. eodag/plugins/search/cop_marine.py +87 -56
  27. eodag/plugins/search/csw.py +47 -37
  28. eodag/plugins/search/qssearch.py +653 -429
  29. eodag/plugins/search/stac_list_assets.py +1 -1
  30. eodag/plugins/search/static_stac_search.py +43 -44
  31. eodag/resources/{product_types.yml → collections.yml} +2594 -2453
  32. eodag/resources/ext_collections.json +1 -1
  33. eodag/resources/ext_product_types.json +1 -1
  34. eodag/resources/providers.yml +2706 -2733
  35. eodag/resources/stac_provider.yml +50 -92
  36. eodag/resources/user_conf_template.yml +9 -0
  37. eodag/types/__init__.py +2 -0
  38. eodag/types/queryables.py +70 -91
  39. eodag/types/search_args.py +1 -1
  40. eodag/utils/__init__.py +97 -21
  41. eodag/utils/dates.py +0 -12
  42. eodag/utils/exceptions.py +6 -6
  43. eodag/utils/free_text_search.py +3 -3
  44. eodag/utils/repr.py +2 -0
  45. {eodag-3.10.1.dist-info → eodag-4.0.0a2.dist-info}/METADATA +13 -99
  46. eodag-4.0.0a2.dist-info/RECORD +93 -0
  47. {eodag-3.10.1.dist-info → eodag-4.0.0a2.dist-info}/entry_points.txt +0 -4
  48. eodag/plugins/authentication/oauth.py +0 -60
  49. eodag/plugins/download/creodias_s3.py +0 -71
  50. eodag/plugins/download/s3rest.py +0 -351
  51. eodag/plugins/search/data_request_search.py +0 -565
  52. eodag/resources/stac.yml +0 -294
  53. eodag/resources/stac_api.yml +0 -2105
  54. eodag/rest/__init__.py +0 -24
  55. eodag/rest/cache.py +0 -70
  56. eodag/rest/config.py +0 -67
  57. eodag/rest/constants.py +0 -26
  58. eodag/rest/core.py +0 -764
  59. eodag/rest/errors.py +0 -210
  60. eodag/rest/server.py +0 -604
  61. eodag/rest/server.wsgi +0 -6
  62. eodag/rest/stac.py +0 -1032
  63. eodag/rest/templates/README +0 -1
  64. eodag/rest/types/__init__.py +0 -18
  65. eodag/rest/types/collections_search.py +0 -44
  66. eodag/rest/types/eodag_search.py +0 -386
  67. eodag/rest/types/queryables.py +0 -174
  68. eodag/rest/types/stac_search.py +0 -272
  69. eodag/rest/utils/__init__.py +0 -207
  70. eodag/rest/utils/cql_evaluate.py +0 -119
  71. eodag/rest/utils/rfc3339.py +0 -64
  72. eodag-3.10.1.dist-info/RECORD +0 -116
  73. {eodag-3.10.1.dist-info → eodag-4.0.0a2.dist-info}/WHEEL +0 -0
  74. {eodag-3.10.1.dist-info → eodag-4.0.0a2.dist-info}/licenses/LICENSE +0 -0
  75. {eodag-3.10.1.dist-info → eodag-4.0.0a2.dist-info}/top_level.txt +0 -0
@@ -31,10 +31,8 @@ import orjson
31
31
  from dateutil.parser import isoparse
32
32
  from dateutil.tz import tzutc
33
33
  from dateutil.utils import today
34
- from pydantic import Field
35
34
  from pydantic.fields import FieldInfo
36
35
  from requests.auth import AuthBase
37
- from shapely.geometry.base import BaseGeometry
38
36
  from typing_extensions import get_args # noqa: F401
39
37
 
40
38
  from eodag.api.product import EOProduct
@@ -47,7 +45,7 @@ from eodag.api.product.metadata_mapping import (
47
45
  mtd_cfg_as_conversion_and_querypath,
48
46
  properties_from_json,
49
47
  )
50
- from eodag.api.search_result import RawSearchResult
48
+ from eodag.api.search_result import RawSearchResult, SearchResult
51
49
  from eodag.plugins.search import PreparedSearch
52
50
  from eodag.plugins.search.qssearch import PostJsonSearch, QueryStringSearch
53
51
  from eodag.types import json_field_definition_to_python # noqa: F401
@@ -57,16 +55,13 @@ from eodag.utils import (
57
55
  DEFAULT_SEARCH_TIMEOUT,
58
56
  deepcopy,
59
57
  dict_items_recursive_sort,
58
+ format_string,
60
59
  get_geometry_from_ecmwf_area,
61
60
  get_geometry_from_ecmwf_feature,
62
61
  get_geometry_from_various,
63
62
  )
64
63
  from eodag.utils.cache import instance_cached_method
65
- from eodag.utils.dates import (
66
- COMPACT_DATE_RANGE_PATTERN,
67
- DATE_RANGE_PATTERN,
68
- is_range_in_range,
69
- )
64
+ from eodag.utils.dates import is_range_in_range
70
65
  from eodag.utils.exceptions import DownloadError, NotAvailableError, ValidationError
71
66
  from eodag.utils.requests import fetch_json
72
67
 
@@ -150,7 +145,6 @@ ECMWF_KEYWORDS = {
150
145
  COP_DS_KEYWORDS = {
151
146
  "aerosol_type",
152
147
  "altitude",
153
- "product_type",
154
148
  "area",
155
149
  "band",
156
150
  "cdr_type",
@@ -183,13 +177,13 @@ COP_DS_KEYWORDS = {
183
177
  "pressure_level",
184
178
  "processing_level",
185
179
  "processing_type",
180
+ "product_type",
186
181
  "product_version",
187
182
  "quantity",
188
183
  "rcm",
189
184
  "region",
190
185
  "release_version",
191
186
  "satellite",
192
- "satellite_mission",
193
187
  "sensor",
194
188
  "sensor_and_algorithm",
195
189
  "soil_level",
@@ -209,9 +203,9 @@ COP_DS_KEYWORDS = {
209
203
 
210
204
  ALLOWED_KEYWORDS = ECMWF_KEYWORDS | COP_DS_KEYWORDS
211
205
 
212
- END = "completionTimeFromAscendingNode"
206
+ END = "end_datetime"
213
207
 
214
- START = "startTimeFromAscendingNode"
208
+ START = "start_datetime"
215
209
 
216
210
 
217
211
  def ecmwf_mtd() -> dict[str, Any]:
@@ -295,9 +289,22 @@ def _update_properties_from_element(
295
289
  prop["description"] = description
296
290
 
297
291
 
298
- def ecmwf_format(v: str) -> str:
299
- """Add ECMWF prefix to value v if v is a ECMWF keyword."""
300
- return ECMWF_PREFIX + v if v in ALLOWED_KEYWORDS else v
292
+ def ecmwf_format(v: str, alias: bool = True) -> str:
293
+ """Add ECMWF prefix to value v if v is a ECMWF keyword.
294
+
295
+ :param v: parameter to format
296
+ :param alias: whether to format for alias (with ':') or for query param (False, with '_')
297
+ :return: formatted parameter
298
+
299
+ >>> ecmwf_format('dataset', alias=False)
300
+ 'ecmwf_dataset'
301
+ >>> ecmwf_format('variable')
302
+ 'ecmwf:variable'
303
+ >>> ecmwf_format('unknown_param')
304
+ 'unknown_param'
305
+ """
306
+ separator = ":" if alias else "_"
307
+ return f"{ECMWF_PREFIX[:-1]}{separator}{v}" if v in ALLOWED_KEYWORDS else v
301
308
 
302
309
 
303
310
  def get_min_max(
@@ -314,11 +321,11 @@ def append_time(input_date: date, time: Optional[str]) -> datetime:
314
321
  """
315
322
  Parses a time string in format HHMM and appends it to a date.
316
323
 
317
- if the time string is in format HH:MM we convert it to HHMM
324
+ if the time string is in format HH:MM or HH_MM we convert it to HHMM
318
325
  """
319
326
  if not time:
320
327
  time = "0000"
321
- time = time.replace(":", "")
328
+ time = re.sub(":|_", "", time)
322
329
  if time == "2400":
323
330
  time = "0000"
324
331
  dt = datetime.combine(input_date, datetime.strptime(time, "%H%M").time())
@@ -444,16 +451,39 @@ class ECMWFSearch(PostJsonSearch):
444
451
  used to parse metadata but that must not be included to the query
445
452
  * :attr:`~eodag.config.PluginConfig.end_date_excluded` (``bool``): Set to `False` if
446
453
  provider does not include end date to search
454
+ * :attr:`~eodag.config.PluginConfig.dates_required` (``bool``): if date parameters are mandatory in the request
447
455
  * :attr:`~eodag.config.PluginConfig.discover_queryables`
448
456
  (:class:`~eodag.config.PluginConfig.DiscoverQueryables`): configuration to fetch the queryables from a
449
457
  provider queryables endpoint; It has the following keys:
450
458
 
451
459
  * :attr:`~eodag.config.PluginConfig.DiscoverQueryables.fetch_url` (``str``): url to fetch the queryables valid
452
- for all product types
453
- * :attr:`~eodag.config.PluginConfig.DiscoverQueryables.product_type_fetch_url` (``str``): url to fetch the
454
- queryables for a specific product type
460
+ for all collections
461
+ * :attr:`~eodag.config.PluginConfig.DiscoverQueryables.collection_fetch_url` (``str``): url to fetch the
462
+ queryables for a specific collection
455
463
  * :attr:`~eodag.config.PluginConfig.DiscoverQueryables.constraints_url` (``str``): url of the constraint file
456
464
  used to build queryables
465
+
466
+ * :attr:`~eodag.config.PluginConfig.dynamic_discover_queryables`
467
+ (``list`` [:class:`~eodag.config.PluginConfig.DynamicDiscoverQueryables`]): list of configurations to fetch
468
+ the queryables from different provider queryables endpoints. A configuration is used based on the given
469
+ selection criterias. The first match is used. If no match is found, it falls back to standard behaviors
470
+ (e.g. discovery using :attr:`~eodag.config.PluginConfig.discover_queryables`).
471
+ Each element of the list has the following keys:
472
+
473
+ * :attr:`~eodag.config.PluginConfig.DynamicDiscoverQueryables.collection_selector`
474
+ (``list`` [:class:`~eodag.config.PluginConfig.CollectionSelector`]): list of collection selection
475
+ criterias. The configuration given in
476
+ :attr:`~eodag.config.PluginConfig.DynamicDiscoverQueryables.discover_queryables` is used if any collection
477
+ selector matches the search parameters. The selector matches if the field value starts with the given
478
+ prefix, i.e. it matches if ``parameters[field].startswith(prefix)==True``. It has the following keys:
479
+
480
+ * :attr:`~eodag.config.PluginConfig.CollectionSelector.field` (``str``) Field in the search parameters to
481
+ match
482
+ * :attr:`~eodag.config.PluginConfig.CollectionSelector.prefix` (``str``) Prefix to match in the field
483
+
484
+ * :attr:`~eodag.config.PluginConfig.DynamicDiscoverQueryables.discover_queryables`
485
+ (``list`` [:class:`~eodag.config.PluginConfig.DiscoverQueryables`]): same as
486
+ :attr:`~eodag.config.PluginConfig.discover_queryables` above.
457
487
  """
458
488
 
459
489
  def __init__(self, provider: str, config: PluginConfig) -> None:
@@ -462,10 +492,10 @@ class ECMWFSearch(PostJsonSearch):
462
492
  **{
463
493
  "id": "$.id",
464
494
  "title": "$.id",
465
- "storageStatus": OFFLINE_STATUS,
466
- "downloadLink": "$.null",
495
+ "order:status": OFFLINE_STATUS,
496
+ "eodag:download_link": "$.null",
467
497
  "geometry": ["feature", "$.geometry"],
468
- "defaultGeometry": "POLYGON((180 -90, 180 90, -180 90, -180 -90, 180 -90))",
498
+ "eodag:default_geometry": "POLYGON((180 -90, 180 90, -180 90, -180 -90, 180 -90))",
469
499
  },
470
500
  **config.metadata_mapping,
471
501
  }
@@ -487,7 +517,9 @@ class ECMWFSearch(PostJsonSearch):
487
517
  },
488
518
  )
489
519
 
490
- def do_search(self, *args: Any, **kwargs: Any) -> list[dict[str, Any]]:
520
+ def do_search(
521
+ self, prep: PreparedSearch = PreparedSearch(items_per_page=None), **kwargs: Any
522
+ ) -> RawSearchResult:
491
523
  """Should perform the actual search request.
492
524
 
493
525
  :param args: arguments to be used in the search
@@ -495,39 +527,47 @@ class ECMWFSearch(PostJsonSearch):
495
527
  :return: list containing the results from the provider in json format
496
528
  """
497
529
  # no real search. We fake it all
498
- return [{}]
530
+ raw_search_results = RawSearchResult([{}])
531
+ raw_search_results.search_params = kwargs
532
+ raw_search_results.query_params = (
533
+ prep.query_params if hasattr(prep, "query_params") else {}
534
+ )
535
+ raw_search_results.collection_def_params = (
536
+ prep.collection_def_params if hasattr(prep, "collection_def_params") else {}
537
+ )
538
+ return raw_search_results
499
539
 
500
540
  def query(
501
541
  self,
502
542
  prep: PreparedSearch = PreparedSearch(),
503
543
  **kwargs: Any,
504
- ) -> tuple[list[EOProduct], Optional[int]]:
544
+ ) -> SearchResult:
505
545
  """Build ready-to-download SearchResult
506
546
 
507
547
  :param prep: :class:`~eodag.plugins.search.PreparedSearch` object containing information needed for the search
508
548
  :param kwargs: keyword arguments to be used in the search
509
549
  :returns: list of products and number of products (optional)
510
550
  """
511
- product_type = prep.product_type
512
- if not product_type:
513
- product_type = kwargs.get("productType")
514
- kwargs = self._preprocess_search_params(kwargs, product_type)
515
- result, num_items = super().query(prep, **kwargs)
516
- if prep.count and not num_items:
517
- num_items = 1
551
+ collection = prep.collection
552
+ if not collection:
553
+ collection = kwargs.get("collection")
554
+ kwargs = self._preprocess_search_params(kwargs, collection)
555
+ result = super().query(prep, **kwargs)
556
+ if prep.count and not result.number_matched:
557
+ result.number_matched = 1
518
558
 
519
- return result, num_items
559
+ return result
520
560
 
521
561
  def clear(self) -> None:
522
562
  """Clear search context"""
523
563
  super().clear()
524
564
 
525
565
  def build_query_string(
526
- self, product_type: str, query_dict: dict[str, Any]
566
+ self, collection: str, query_dict: dict[str, Any]
527
567
  ) -> tuple[dict[str, Any], str]:
528
568
  """Build The query string using the search parameters
529
569
 
530
- :param product_type: product type id
570
+ :param collection: collection id
531
571
  :param query_dict: keyword arguments to be used in the query string
532
572
  :return: formatted query params and encode query string
533
573
  """
@@ -542,21 +582,21 @@ class ECMWFSearch(PostJsonSearch):
542
582
  ordered_kwargs.update(query_dict)
543
583
 
544
584
  return super().build_query_string(
545
- product_type=product_type, query_dict=ordered_kwargs
585
+ collection=collection, query_dict=ordered_kwargs
546
586
  )
547
587
 
548
588
  def _preprocess_search_params(
549
- self, params: dict[str, Any], product_type: Optional[str]
589
+ self, params: dict[str, Any], collection: Optional[str]
550
590
  ) -> dict[str, Any]:
551
591
  """Preprocess search parameters before making a request to the CDS API.
552
592
 
553
593
  This method is responsible for checking and updating the provided search parameters
554
- to ensure that required parameters like 'productType', 'startTimeFromAscendingNode',
555
- 'completionTimeFromAscendingNode', and 'geometry' are properly set. If not specified
594
+ to ensure that required parameters like 'collection', 'start_datetime',
595
+ 'end_datetime', and 'geometry' are properly set. If not specified
556
596
  in the input parameters, default values or values from the configuration are used.
557
597
 
558
598
  :param params: Search parameters to be preprocessed.
559
- :param product_type: (optional) product type id
599
+ :param collection: (optional) collection id
560
600
  """
561
601
 
562
602
  _dc_qs = params.get("_dc_qs")
@@ -577,13 +617,15 @@ class ECMWFSearch(PostJsonSearch):
577
617
  params["geometry"] = _dc_qp["area"].split("/")
578
618
 
579
619
  params = {
580
- k.removeprefix(ECMWF_PREFIX): v for k, v in params.items() if v is not None
620
+ k.removeprefix(ECMWF_PREFIX).removeprefix(f"{ECMWF_PREFIX[:-1]}_"): v
621
+ for k, v in params.items()
622
+ if v is not None
581
623
  }
582
624
 
583
625
  # dates
584
626
  # check if default dates have to be added
585
627
  if getattr(self.config, "dates_required", False):
586
- self._check_date_params(params, product_type)
628
+ self._check_date_params(params, collection)
587
629
 
588
630
  # adapt end date if it is midnight
589
631
  if END in params:
@@ -628,84 +670,101 @@ class ECMWFSearch(PostJsonSearch):
628
670
  return params
629
671
 
630
672
  def _check_date_params(
631
- self, keywords: dict[str, Any], product_type: Optional[str]
673
+ self, keywords: dict[str, Any], collection: Optional[str]
632
674
  ) -> None:
633
675
  """checks if start and end date are present in the keywords and adds them if not"""
634
676
 
635
- if START and END in keywords:
677
+ if START in keywords and END in keywords:
636
678
  return
637
679
 
638
- product_type_conf = getattr(self.config, "metadata_mapping", {})
680
+ collection_conf = getattr(self.config, "metadata_mapping", {})
639
681
  if (
640
- product_type
641
- and product_type in self.config.products
642
- and "metadata_mapping" in self.config.products[product_type]
682
+ collection
683
+ and collection in self.config.products
684
+ and "metadata_mapping" in self.config.products[collection]
643
685
  ):
644
- product_type_conf = self.config.products[product_type]["metadata_mapping"]
686
+ collection_conf = self.config.products[collection]["metadata_mapping"]
645
687
 
646
688
  # start time given, end time missing
647
689
  if START in keywords:
648
690
  keywords[END] = (
649
691
  keywords[START]
650
- if END in product_type_conf
651
- else self.get_product_type_cfg_value(
652
- "missionEndDate", today().isoformat()
653
- )
692
+ if END in collection_conf
693
+ # else self.get_collection_cfg_value(
694
+ else self.get_collection_cfg_dates(None, today().isoformat())[1]
654
695
  )
655
696
  return
656
697
 
657
- if END in product_type_conf:
658
- mapping = product_type_conf[START]
698
+ if END in collection_conf:
699
+ mapping = collection_conf[START]
659
700
  if not isinstance(mapping, list):
660
- mapping = product_type_conf[END]
701
+ mapping = collection_conf[END]
661
702
  if isinstance(mapping, list):
662
703
  # if startTime is not given but other time params (e.g. year/month/(day)) are given,
663
704
  # no default date is required
664
705
  start, end = ecmwf_temporal_to_eodag(keywords)
665
706
  if start is None:
666
- keywords[START] = self.get_product_type_cfg_value(
667
- "missionStartDate", DEFAULT_MISSION_START_DATE
707
+ col_start, col_end = self.get_collection_cfg_dates(
708
+ DEFAULT_MISSION_START_DATE, today().isoformat()
668
709
  )
710
+ keywords[START] = col_start
669
711
  keywords[END] = (
670
- keywords[START]
671
- if END in product_type_conf
672
- else self.get_product_type_cfg_value(
673
- "missionEndDate", today().isoformat()
674
- )
712
+ keywords[START] if END in collection_conf else col_end
675
713
  )
676
714
  else:
677
715
  keywords[START] = start
678
716
  keywords[END] = end
679
717
 
680
- def _get_product_type_queryables(
681
- self, product_type: Optional[str], alias: Optional[str], filters: dict[str, Any]
718
+ def _get_collection_queryables(
719
+ self, collection: Optional[str], alias: Optional[str], filters: dict[str, Any]
682
720
  ) -> QueryablesDict:
683
721
  """Override to set additional_properties to false."""
684
722
  default_values: dict[str, Any] = deepcopy(
685
- getattr(self.config, "products", {}).get(product_type, {})
723
+ getattr(self.config, "products", {}).get(collection, {})
686
724
  )
687
725
  default_values.pop("metadata_mapping", None)
726
+ default_values.pop("metadata_mapping_from_product", None)
688
727
 
689
- filters["productType"] = product_type
728
+ filters["collection"] = collection
690
729
  queryables = self.discover_queryables(**{**default_values, **filters}) or {}
691
730
 
692
731
  return QueryablesDict(additional_properties=False, **queryables)
693
732
 
733
+ def _find_dynamic_queryables_config(
734
+ self, kwargs: dict[str, Any], dynamic_config: list
735
+ ) -> dict[str, Any]:
736
+ """Find the appropriate queryables configuration from dynamic configuration.
737
+
738
+ :param kwargs: Search parameters
739
+ :param dynamic_config: List of dynamic discover queryables configurations
740
+ :return: Found queryables configuration or empty dict
741
+ """
742
+ for dc in dynamic_config:
743
+ for cs in dc["collection_selector"]:
744
+ field = cs["field"]
745
+ if kwargs[field].startswith(cs["prefix"]):
746
+ return dc["discover_queryables"]
747
+ return {}
748
+
694
749
  def discover_queryables(
695
- self, **kwargs: Any
750
+ self,
751
+ **kwargs: Any,
696
752
  ) -> Optional[dict[str, Annotated[Any, FieldInfo]]]:
697
753
  """Fetch queryables list from provider using its constraints file
698
754
 
699
- :param kwargs: additional filters for queryables (`productType` and other search
755
+ :param kwargs: additional filters for queryables (`collection` and other search
700
756
  arguments)
701
757
  :returns: fetched queryable parameters dict
702
758
  """
703
- product_type = kwargs.pop("productType")
759
+ collection = kwargs.pop("collection")
704
760
 
705
- pt_config = self.get_product_type_def_params(product_type)
761
+ col_config = self.get_collection_def_params(collection)
706
762
 
707
- default_values = deepcopy(pt_config)
763
+ default_values = deepcopy(col_config)
708
764
  default_values.pop("metadata_mapping", None)
765
+ default_values.pop("metadata_mapping_from_product", None)
766
+ default_values.pop("discover_queryables", None)
767
+ kwargs.pop("discover_queryables", None)
709
768
  filters = {**default_values, **kwargs}
710
769
 
711
770
  if "start" in filters:
@@ -716,29 +775,33 @@ class ECMWFSearch(PostJsonSearch):
716
775
  # extract default datetime and convert geometry
717
776
  try:
718
777
  processed_filters = self._preprocess_search_params(
719
- deepcopy(filters), product_type
778
+ deepcopy(filters), collection
720
779
  )
721
780
  except Exception as e:
722
781
  raise ValidationError(e.args[0]) from e
723
782
 
724
- constraints_url = format_metadata(
725
- getattr(self.config, "discover_queryables", {}).get("constraints_url", ""),
726
- **filters,
727
- )
783
+ # dynamic_discover_queryables for WekeoECMWFSearch
784
+ queryables_config = {}
785
+ if dynamic_config := getattr(self.config, "dynamic_discover_queryables", []):
786
+ queryables_config = self._find_dynamic_queryables_config(
787
+ kwargs, dynamic_config
788
+ )
789
+
790
+ provider_dq = getattr(self.config, "discover_queryables", {}) or {}
791
+ product_dq = col_config.get("discover_queryables", {}) or {}
792
+ dq_conf = {**provider_dq, **product_dq, **queryables_config}
793
+ constraints_url = format_metadata(dq_conf.get("constraints_url", ""), **filters)
728
794
  constraints: list[dict[str, Any]] = self._fetch_data(constraints_url)
729
795
 
730
- form_url = format_metadata(
731
- getattr(self.config, "discover_queryables", {}).get("form_url", ""),
732
- **filters,
733
- )
796
+ form_url = format_metadata(dq_conf.get("form_url", ""), **filters)
734
797
  form: list[dict[str, Any]] = self._fetch_data(form_url)
735
798
 
736
799
  formated_filters = self.format_as_provider_keyword(
737
- product_type, deepcopy(processed_filters)
800
+ collection, deepcopy(processed_filters)
738
801
  )
739
802
  # we re-apply kwargs input to consider override of year, month, day and time.
740
803
  for k, v in {**default_values, **kwargs}.items():
741
- key = k.removeprefix(ECMWF_PREFIX)
804
+ key = k.removeprefix(ECMWF_PREFIX).removeprefix(f"{ECMWF_PREFIX[:-1]}_")
742
805
 
743
806
  if key not in ALLOWED_KEYWORDS | {
744
807
  START,
@@ -781,28 +844,30 @@ class ECMWFSearch(PostJsonSearch):
781
844
  else:
782
845
  values_url = getattr(self.config, "available_values_url", "")
783
846
  if not values_url:
784
- return self.queryables_from_metadata_mapping(product_type)
847
+ return self.queryables_from_metadata_mapping(collection)
785
848
  if "{" in values_url:
786
- values_url = values_url.format(**filters)
849
+ values_url = format_string(None, values_url, **filters)
787
850
  data = self._fetch_data(values_url)
788
851
  available_values = data["constraints"]
789
852
  required_keywords = data.get("required", [])
790
853
 
791
854
  # To check if all keywords are queryable parameters, we check if they are in the
792
- # available values or the product type config (available values calculated from the
855
+ # available values or the collection config (available values calculated from the
793
856
  # constraints might not include all queryables)
794
857
  for keyword in processed_filters:
795
858
  if (
796
859
  keyword
797
860
  not in available_values.keys()
798
- | pt_config.keys()
861
+ | col_config.keys()
799
862
  | {
800
863
  START,
801
864
  END,
802
865
  "geometry",
803
866
  }
804
867
  and keyword not in [f["name"] for f in form]
805
- and keyword.removeprefix(ECMWF_PREFIX)
868
+ and keyword.removeprefix(ECMWF_PREFIX).removeprefix(
869
+ f"{ECMWF_PREFIX[:-1]}_"
870
+ )
806
871
  not in set(list(available_values.keys()) + [f["name"] for f in form])
807
872
  ):
808
873
  raise ValidationError(
@@ -823,10 +888,11 @@ class ECMWFSearch(PostJsonSearch):
823
888
 
824
889
  # ecmwf:date is replaced by start and end.
825
890
  # start and end filters are supported whenever combinations of "year", "month", "day" filters exist
891
+ queryable_prefix = f"{ECMWF_PREFIX[:-1]}_"
826
892
  if (
827
- queryables.pop(f"{ECMWF_PREFIX}date", None)
828
- or f"{ECMWF_PREFIX}year" in queryables
829
- or f"{ECMWF_PREFIX}hyear" in queryables
893
+ queryables.pop(f"{queryable_prefix}date", None)
894
+ or f"{queryable_prefix}year" in queryables
895
+ or f"{queryable_prefix}hyear" in queryables
830
896
  ):
831
897
  queryables.update(
832
898
  {
@@ -842,13 +908,7 @@ class ECMWFSearch(PostJsonSearch):
842
908
 
843
909
  # area is geom in EODAG.
844
910
  if queryables.pop("area", None):
845
- queryables["geom"] = Annotated[
846
- Union[str, dict[str, float], BaseGeometry],
847
- Field(
848
- None,
849
- description="Read EODAG documentation for all supported geometry format.",
850
- ),
851
- ]
911
+ queryables["geom"] = Queryables.get_with_default("geom", None)
852
912
 
853
913
  return queryables
854
914
 
@@ -899,21 +959,13 @@ class ECMWFSearch(PostJsonSearch):
899
959
  )
900
960
 
901
961
  # We convert every single value to a list of string
902
- filter_v = list(values) if isinstance(values, tuple) else values
903
- filter_v = filter_v if isinstance(filter_v, list) else [filter_v]
962
+ filter_v = values if isinstance(values, (list, tuple)) else [values]
904
963
 
905
964
  # We strip values of superfluous quotes (added by mapping converter to_geojson).
906
- # ECMWF accept date ranges with /to/. We need to split it to an array
907
- # ECMWF accept date ranges in format val1/val2. We need to split it to an array
908
- date_regex = [
909
- re.compile(p) for p in (DATE_RANGE_PATTERN, COMPACT_DATE_RANGE_PATTERN)
910
- ]
911
- is_date = any(
912
- any(r.match(v) is not None for r in date_regex) for v in filter_v
913
- )
914
- if is_date:
915
- sep = re.compile(r"/to/|/")
916
- filter_v = [i for v in filter_v for i in sep.split(str(v))]
965
+ # ECMWF accept values with /to/. We need to split it to an array
966
+ # ECMWF accept values in format val1/val2. We need to split it to an array
967
+ sep = re.compile(r"/to/|/")
968
+ filter_v = [i for v in filter_v for i in sep.split(str(v))]
917
969
 
918
970
  # special handling for time 0000 converted to 0 by pre-formating with metadata_mapping
919
971
  if keyword.split(":")[-1] == "time":
@@ -1048,16 +1100,21 @@ class ECMWFSearch(PostJsonSearch):
1048
1100
  if default and prop.get("type") == "string" and isinstance(default, list):
1049
1101
  default = ",".join(default)
1050
1102
 
1051
- is_required = bool(element.get("required"))
1103
+ is_required = bool(element.get("required")) and bool(
1104
+ available_values.get(name)
1105
+ )
1052
1106
  if is_required:
1053
1107
  required_list.append(name)
1054
1108
 
1055
- queryables[ecmwf_format(name)] = Annotated[
1109
+ formatted_param = ecmwf_format(name, alias=False)
1110
+ formatted_alias = ecmwf_format(name)
1111
+ queryables[formatted_param] = Annotated[
1056
1112
  get_args(
1057
1113
  json_field_definition_to_python(
1058
1114
  prop,
1059
1115
  default_value=default,
1060
1116
  required=is_required,
1117
+ alias=formatted_alias,
1061
1118
  )
1062
1119
  )
1063
1120
  ]
@@ -1087,14 +1144,16 @@ class ECMWFSearch(PostJsonSearch):
1087
1144
  for name, values in available_values.items():
1088
1145
  # Rename keywords from form with metadata mapping.
1089
1146
  # Needed to map constraints like "xxxx" to eodag parameter "ecmwf:xxxx"
1090
- key = ecmwf_format(name)
1147
+ formatted_param = ecmwf_format(name, alias=False)
1148
+ formatted_alias = ecmwf_format(name)
1091
1149
 
1092
- queryables[key] = Annotated[
1150
+ queryables[formatted_param] = Annotated[
1093
1151
  get_args(
1094
1152
  json_field_definition_to_python(
1095
1153
  {"type": "string", "title": name, "enum": values},
1096
1154
  default_value=defaults.get(name),
1097
- required=bool(key in required),
1155
+ required=bool(formatted_alias in required),
1156
+ alias=formatted_alias,
1098
1157
  )
1099
1158
  )
1100
1159
  ]
@@ -1102,32 +1161,32 @@ class ECMWFSearch(PostJsonSearch):
1102
1161
  return queryables
1103
1162
 
1104
1163
  def format_as_provider_keyword(
1105
- self, product_type: str, properties: dict[str, Any]
1164
+ self, collection: str, properties: dict[str, Any]
1106
1165
  ) -> dict[str, Any]:
1107
1166
  """Return provider equivalent keyword names from EODAG keywords.
1108
1167
 
1109
- :param product_type: product type id
1168
+ :param collection: collection id
1110
1169
  :param properties: dict of properties to be formatted
1111
1170
  :return: dict of formatted properties
1112
1171
  """
1113
- properties["productType"] = product_type
1172
+ properties["collection"] = collection
1114
1173
 
1115
- # provider product type specific conf
1116
- product_type_def_params = self.get_product_type_def_params(
1117
- product_type, format_variables=properties
1174
+ # provider collection specific conf
1175
+ collection_def_params = self.get_collection_def_params(
1176
+ collection, format_variables=properties
1118
1177
  )
1119
1178
 
1120
- # Add to the query, the queryable parameters set in the provider product type definition
1179
+ # Add to the query, the queryable parameters set in the provider collection definition
1121
1180
  properties.update(
1122
1181
  {
1123
1182
  k: v
1124
- for k, v in product_type_def_params.items()
1183
+ for k, v in collection_def_params.items()
1125
1184
  if k not in properties.keys()
1126
1185
  and k in self.config.metadata_mapping.keys()
1127
1186
  and isinstance(self.config.metadata_mapping[k], list)
1128
1187
  }
1129
1188
  )
1130
- qp, _ = self.build_query_string(product_type, properties)
1189
+ qp, _ = self.build_query_string(collection, properties)
1131
1190
 
1132
1191
  return qp
1133
1192
 
@@ -1160,7 +1219,7 @@ class ECMWFSearch(PostJsonSearch):
1160
1219
  :returns: list of single :class:`~eodag.api.product._product.EOProduct`
1161
1220
  """
1162
1221
 
1163
- product_type = kwargs.get("productType")
1222
+ collection = kwargs.get("collection")
1164
1223
 
1165
1224
  result = results[0]
1166
1225
 
@@ -1178,27 +1237,27 @@ class ECMWFSearch(PostJsonSearch):
1178
1237
 
1179
1238
  if result:
1180
1239
  properties = result
1181
- properties.update(result.pop("request_params", None) or {})
1240
+ properties.update(result.pop("eodag:request_params", None) or {})
1182
1241
 
1183
1242
  properties = {k: v for k, v in properties.items() if not k.startswith("__")}
1184
1243
 
1185
1244
  properties["geometry"] = properties.get("area") or DEFAULT_GEOMETRY
1186
1245
 
1187
1246
  start, end = ecmwf_temporal_to_eodag(properties)
1188
- properties["startTimeFromAscendingNode"] = start
1189
- properties["completionTimeFromAscendingNode"] = end
1247
+ properties["start_datetime"] = start
1248
+ properties["end_datetime"] = end
1190
1249
 
1191
1250
  else:
1192
1251
  # use all available query_params to parse properties
1193
1252
  result_data: dict[str, Any] = {
1194
- **results.product_type_def_params,
1253
+ **results.collection_def_params,
1195
1254
  **sorted_unpaginated_qp,
1196
1255
  **{"qs": sorted_unpaginated_qp},
1197
1256
  }
1198
1257
 
1199
- # update result with product_type_def_params and search args if not None (and not auth)
1258
+ # update result with collection_def_params and search args if not None (and not auth)
1200
1259
  kwargs.pop("auth", None)
1201
- result_data.update(results.product_type_def_params)
1260
+ result_data.update(results.collection_def_params)
1202
1261
  result_data = {
1203
1262
  **result_data,
1204
1263
  **{k: v for k, v in kwargs.items() if v is not None},
@@ -1213,17 +1272,18 @@ class ECMWFSearch(PostJsonSearch):
1213
1272
  query_hash = hashlib.sha1(str(result_data).encode("UTF-8")).hexdigest()
1214
1273
 
1215
1274
  properties["title"] = properties["id"] = (
1216
- (product_type or kwargs.get("dataset", self.provider)).upper()
1275
+ (collection or kwargs.get("dataset", self.provider)).upper()
1217
1276
  + "_ORDERABLE_"
1218
1277
  + query_hash
1219
1278
  )
1220
- # use product_type_config as default properties
1221
- product_type_config = getattr(self.config, "product_type_config", {})
1222
- properties = dict(product_type_config, **properties)
1279
+
1280
+ # collection alias (required by opentelemetry-instrumentation-eodag)
1281
+ if alias := getattr(self.config, "collection_config", {}).get("alias"):
1282
+ properties["eodag:alias"] = alias
1223
1283
 
1224
1284
  qs = geojson.dumps(sorted_unpaginated_qp)
1225
1285
 
1226
- # used by server mode to generate downloadlink href
1286
+ # used by server mode to generate eodag:download_link href
1227
1287
  # TODO: to remove once the legacy server is removed
1228
1288
  properties["_dc_qs"] = quote_plus(qs)
1229
1289
 
@@ -1272,12 +1332,12 @@ def _check_id(product: EOProduct) -> EOProduct:
1272
1332
  if not on_response_mm:
1273
1333
  return product
1274
1334
 
1275
- logger.debug(f"Update product properties using given orderId {product_id}")
1335
+ logger.debug(f"Update product properties using given eodag:order_id {product_id}")
1276
1336
  on_response_mm_jsonpath = mtd_cfg_as_conversion_and_querypath(
1277
1337
  on_response_mm,
1278
1338
  )
1279
1339
  properties_update = properties_from_json(
1280
- {}, {**on_response_mm_jsonpath, **{"orderId": (None, product_id)}}
1340
+ {}, {**on_response_mm_jsonpath, **{"eodag:order_id": (None, product_id)}}
1281
1341
  )
1282
1342
  product.properties.update(
1283
1343
  {k: v for k, v in properties_update.items() if v != NOT_AVAILABLE}
@@ -1290,7 +1350,7 @@ def _check_id(product: EOProduct) -> EOProduct:
1290
1350
  product.downloader._order_status(product=product, auth=auth) # type: ignore
1291
1351
  # when a NotAvailableError is catched, it means the product is not ready and still needs to be polled
1292
1352
  except NotAvailableError:
1293
- product.properties["storageStatus"] = STAGING_STATUS
1353
+ product.properties["order:status"] = STAGING_STATUS
1294
1354
  except Exception as e:
1295
1355
  if (
1296
1356
  isinstance(e, DownloadError) or isinstance(e, ValidationError)
@@ -1302,16 +1362,16 @@ def _check_id(product: EOProduct) -> EOProduct:
1302
1362
 
1303
1363
  # update product id
1304
1364
  product.properties["id"] = product_id
1305
- # update product type if needed
1306
- if product.product_type is None:
1307
- product.product_type = product.properties.get("ecmwf:dataset")
1365
+ # update collection if needed
1366
+ if product.collection is None:
1367
+ product.collection = product.properties.get("ecmwf:dataset")
1308
1368
  # update product title
1309
1369
  product.properties["title"] = (
1310
- (product.product_type or product.provider).upper() + "_" + product_id
1370
+ (product.collection or product.provider).upper() + "_" + product_id
1311
1371
  )
1312
- # use NOT_AVAILABLE as fallback product_type to avoid using guess_product_type
1313
- if product.product_type is None:
1314
- product.product_type = NOT_AVAILABLE
1372
+ # use NOT_AVAILABLE as fallback collection to avoid using guess_collection
1373
+ if product.collection is None:
1374
+ product.collection = NOT_AVAILABLE
1315
1375
 
1316
1376
  return product
1317
1377
 
@@ -1371,7 +1431,7 @@ class MeteoblueSearch(ECMWFSearch):
1371
1431
 
1372
1432
  def do_search(
1373
1433
  self, prep: PreparedSearch = PreparedSearch(items_per_page=None), **kwargs: Any
1374
- ) -> list[dict[str, Any]]:
1434
+ ) -> RawSearchResult:
1375
1435
  """Perform the actual search request, and return result in a single element.
1376
1436
 
1377
1437
  :param prep: :class:`~eodag.plugins.search.PreparedSearch` object containing information for the search
@@ -1386,19 +1446,23 @@ class MeteoblueSearch(ECMWFSearch):
1386
1446
  f" {self.__class__.__name__} instance"
1387
1447
  )
1388
1448
  response = self._request(prep)
1449
+ raw_search_results = RawSearchResult([response.json()])
1450
+ raw_search_results.search_params = kwargs
1389
1451
 
1390
- return [response.json()]
1452
+ raw_search_results.query_params = prep.query_params
1453
+ raw_search_results.collection_def_params = prep.collection_def_params
1454
+ return raw_search_results
1391
1455
 
1392
1456
  def build_query_string(
1393
- self, product_type: str, query_dict: dict[str, Any]
1457
+ self, collection: str, query_dict: dict[str, Any]
1394
1458
  ) -> tuple[dict[str, Any], str]:
1395
1459
  """Build The query string using the search parameters
1396
1460
 
1397
- :param product_type: product type id
1461
+ :param collection: collection id
1398
1462
  :param query_dict: keyword arguments to be used in the query string
1399
1463
  :return: formatted query params and encode query string
1400
1464
  """
1401
- return QueryStringSearch.build_query_string(self, product_type, query_dict)
1465
+ return QueryStringSearch.build_query_string(self, collection, query_dict)
1402
1466
 
1403
1467
  def normalize_results(self, results, **kwargs):
1404
1468
  """Build :class:`~eodag.api.product._product.EOProduct` from provider result
@@ -1408,7 +1472,7 @@ class MeteoblueSearch(ECMWFSearch):
1408
1472
  :returns: list of single :class:`~eodag.api.product._product.EOProduct`
1409
1473
  """
1410
1474
 
1411
- product_type = kwargs.get("productType")
1475
+ collection = kwargs.get("collection")
1412
1476
 
1413
1477
  result = results[0]
1414
1478
 
@@ -1442,9 +1506,9 @@ class MeteoblueSearch(ECMWFSearch):
1442
1506
 
1443
1507
  query_hash = hashlib.sha1(str(qs).encode("UTF-8")).hexdigest()
1444
1508
 
1445
- # update result with product_type_def_params and search args if not None (and not auth)
1509
+ # update result with collection_def_params and search args if not None (and not auth)
1446
1510
  kwargs.pop("auth", None)
1447
- result.update(results.product_type_def_params)
1511
+ result.update(results.collection_def_params)
1448
1512
  result = dict(result, **{k: v for k, v in kwargs.items() if v is not None})
1449
1513
 
1450
1514
  # parse properties
@@ -1454,17 +1518,16 @@ class MeteoblueSearch(ECMWFSearch):
1454
1518
  discovery_config=getattr(self.config, "discover_metadata", {}),
1455
1519
  )
1456
1520
 
1457
- properties = {
1458
- # use product_type_config as default properties
1459
- **getattr(self.config, "product_type_config", {}),
1460
- **{ecmwf_format(k): v for k, v in parsed_properties.items()},
1461
- }
1521
+ properties = {ecmwf_format(k): v for k, v in parsed_properties.items()}
1522
+ # collection alias (required by opentelemetry-instrumentation-eodag)
1523
+ if alias := getattr(self.config, "collection_config", {}).get("alias"):
1524
+ properties["eodag:alias"] = alias
1462
1525
 
1463
1526
  def slugify(date_str: str) -> str:
1464
1527
  return date_str.split("T")[0].replace("-", "")
1465
1528
 
1466
1529
  # build product id
1467
- product_id = (product_type or self.provider).upper()
1530
+ product_id = (collection or self.provider).upper()
1468
1531
 
1469
1532
  start = properties.get(START, NOT_AVAILABLE)
1470
1533
  end = properties.get(END, NOT_AVAILABLE)
@@ -1478,17 +1541,14 @@ class MeteoblueSearch(ECMWFSearch):
1478
1541
 
1479
1542
  properties["id"] = properties["title"] = product_id
1480
1543
 
1481
- # used by server mode to generate downloadlink href
1544
+ # used by server mode to generate eodag:download_link href
1482
1545
  properties["_dc_qs"] = quote_plus(qs)
1483
1546
 
1484
1547
  product = EOProduct(
1485
1548
  provider=self.provider,
1486
- productType=product_type,
1549
+ collection=collection,
1487
1550
  properties=properties,
1488
1551
  )
1489
- # use product_type_config as default properties
1490
- product_type_config = getattr(self.config, "product_type_config", {})
1491
- product.properties = dict(product_type_config, **product.properties)
1492
1552
 
1493
1553
  return [
1494
1554
  product,
@@ -1534,9 +1594,9 @@ class WekeoECMWFSearch(ECMWFSearch):
1534
1594
  # id is order id (only letters and numbers) -> use parent normalize results
1535
1595
  return super().normalize_results(results, **kwargs)
1536
1596
 
1537
- # formating of orderLink requires access to the productType value.
1597
+ # formating of eodag:order_link requires access to the collection value.
1538
1598
  results.data = [
1539
- {**result, **results.product_type_def_params} for result in results
1599
+ {**result, **results.collection_def_params} for result in results
1540
1600
  ]
1541
1601
 
1542
1602
  normalized = QueryStringSearch.normalize_results(self, results, **kwargs)
@@ -1561,14 +1621,16 @@ class WekeoECMWFSearch(ECMWFSearch):
1561
1621
  dataset = "_".join(splitted_id[:-1])
1562
1622
  query_hash = splitted_id[-1]
1563
1623
  product.properties["title"] = product.properties["id"] = (
1564
- (product.product_type or dataset or self.provider).upper()
1624
+ (product.collection or dataset or self.provider).upper()
1565
1625
  + "_ORDERABLE_"
1566
1626
  + query_hash
1567
1627
  )
1568
1628
 
1569
1629
  return normalized
1570
1630
 
1571
- def do_search(self, *args: Any, **kwargs: Any) -> list[dict[str, Any]]:
1631
+ def do_search(
1632
+ self, prep: PreparedSearch = PreparedSearch(items_per_page=None), **kwargs: Any
1633
+ ) -> RawSearchResult:
1572
1634
  """Should perform the actual search request.
1573
1635
 
1574
1636
  :param args: arguments to be used in the search
@@ -1578,6 +1640,16 @@ class WekeoECMWFSearch(ECMWFSearch):
1578
1640
  if "id" in kwargs and "ORDERABLE" not in kwargs["id"]:
1579
1641
  # id is order id (only letters and numbers) -> use parent normalize results.
1580
1642
  # No real search. We fake it all, then check order status using given id
1581
- return [{}]
1643
+ raw_search_results = RawSearchResult([{}])
1644
+ raw_search_results.search_params = kwargs
1645
+ raw_search_results.query_params = (
1646
+ prep.query_params if hasattr(prep, "query_params") else {}
1647
+ )
1648
+ raw_search_results.collection_def_params = (
1649
+ prep.collection_def_params
1650
+ if hasattr(prep, "collection_def_params")
1651
+ else {}
1652
+ )
1653
+ return raw_search_results
1582
1654
  else:
1583
- return QueryStringSearch.do_search(self, *args, **kwargs)
1655
+ return QueryStringSearch.do_search(self, prep, **kwargs)