eodag 3.0.0b3__py3-none-any.whl → 3.1.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (94) hide show
  1. eodag/api/core.py +347 -247
  2. eodag/api/product/_assets.py +44 -15
  3. eodag/api/product/_product.py +58 -47
  4. eodag/api/product/drivers/__init__.py +81 -4
  5. eodag/api/product/drivers/base.py +65 -4
  6. eodag/api/product/drivers/generic.py +65 -0
  7. eodag/api/product/drivers/sentinel1.py +97 -0
  8. eodag/api/product/drivers/sentinel2.py +95 -0
  9. eodag/api/product/metadata_mapping.py +129 -93
  10. eodag/api/search_result.py +28 -12
  11. eodag/cli.py +61 -24
  12. eodag/config.py +457 -167
  13. eodag/plugins/apis/base.py +10 -4
  14. eodag/plugins/apis/ecmwf.py +53 -23
  15. eodag/plugins/apis/usgs.py +41 -17
  16. eodag/plugins/authentication/aws_auth.py +30 -18
  17. eodag/plugins/authentication/base.py +14 -3
  18. eodag/plugins/authentication/generic.py +14 -3
  19. eodag/plugins/authentication/header.py +14 -6
  20. eodag/plugins/authentication/keycloak.py +44 -25
  21. eodag/plugins/authentication/oauth.py +18 -4
  22. eodag/plugins/authentication/openid_connect.py +192 -171
  23. eodag/plugins/authentication/qsauth.py +12 -4
  24. eodag/plugins/authentication/sas_auth.py +22 -5
  25. eodag/plugins/authentication/token.py +95 -17
  26. eodag/plugins/authentication/token_exchange.py +19 -19
  27. eodag/plugins/base.py +4 -4
  28. eodag/plugins/crunch/base.py +8 -5
  29. eodag/plugins/crunch/filter_date.py +9 -6
  30. eodag/plugins/crunch/filter_latest_intersect.py +9 -8
  31. eodag/plugins/crunch/filter_latest_tpl_name.py +8 -8
  32. eodag/plugins/crunch/filter_overlap.py +9 -11
  33. eodag/plugins/crunch/filter_property.py +10 -10
  34. eodag/plugins/download/aws.py +181 -105
  35. eodag/plugins/download/base.py +49 -67
  36. eodag/plugins/download/creodias_s3.py +40 -2
  37. eodag/plugins/download/http.py +247 -223
  38. eodag/plugins/download/s3rest.py +29 -28
  39. eodag/plugins/manager.py +176 -41
  40. eodag/plugins/search/__init__.py +6 -5
  41. eodag/plugins/search/base.py +123 -60
  42. eodag/plugins/search/build_search_result.py +1046 -355
  43. eodag/plugins/search/cop_marine.py +132 -39
  44. eodag/plugins/search/creodias_s3.py +19 -68
  45. eodag/plugins/search/csw.py +48 -8
  46. eodag/plugins/search/data_request_search.py +124 -23
  47. eodag/plugins/search/qssearch.py +531 -310
  48. eodag/plugins/search/stac_list_assets.py +85 -0
  49. eodag/plugins/search/static_stac_search.py +23 -24
  50. eodag/resources/ext_product_types.json +1 -1
  51. eodag/resources/product_types.yml +1295 -355
  52. eodag/resources/providers.yml +1819 -3010
  53. eodag/resources/stac.yml +3 -163
  54. eodag/resources/stac_api.yml +2 -2
  55. eodag/resources/user_conf_template.yml +115 -99
  56. eodag/rest/cache.py +2 -2
  57. eodag/rest/config.py +3 -4
  58. eodag/rest/constants.py +0 -1
  59. eodag/rest/core.py +157 -117
  60. eodag/rest/errors.py +181 -0
  61. eodag/rest/server.py +57 -339
  62. eodag/rest/stac.py +133 -581
  63. eodag/rest/types/collections_search.py +3 -3
  64. eodag/rest/types/eodag_search.py +41 -30
  65. eodag/rest/types/queryables.py +42 -32
  66. eodag/rest/types/stac_search.py +15 -16
  67. eodag/rest/utils/__init__.py +14 -21
  68. eodag/rest/utils/cql_evaluate.py +6 -6
  69. eodag/rest/utils/rfc3339.py +2 -2
  70. eodag/types/__init__.py +153 -32
  71. eodag/types/bbox.py +2 -2
  72. eodag/types/download_args.py +4 -4
  73. eodag/types/queryables.py +183 -73
  74. eodag/types/search_args.py +6 -6
  75. eodag/types/whoosh.py +127 -3
  76. eodag/utils/__init__.py +228 -106
  77. eodag/utils/exceptions.py +47 -26
  78. eodag/utils/import_system.py +2 -2
  79. eodag/utils/logging.py +37 -77
  80. eodag/utils/repr.py +65 -6
  81. eodag/utils/requests.py +13 -15
  82. eodag/utils/rest.py +2 -2
  83. eodag/utils/s3.py +231 -0
  84. eodag/utils/stac_reader.py +11 -11
  85. {eodag-3.0.0b3.dist-info → eodag-3.1.0.dist-info}/METADATA +81 -81
  86. eodag-3.1.0.dist-info/RECORD +113 -0
  87. {eodag-3.0.0b3.dist-info → eodag-3.1.0.dist-info}/WHEEL +1 -1
  88. {eodag-3.0.0b3.dist-info → eodag-3.1.0.dist-info}/entry_points.txt +5 -2
  89. eodag/resources/constraints/climate-dt.json +0 -13
  90. eodag/resources/constraints/extremes-dt.json +0 -8
  91. eodag/utils/constraints.py +0 -244
  92. eodag-3.0.0b3.dist-info/RECORD +0 -110
  93. {eodag-3.0.0b3.dist-info → eodag-3.1.0.dist-info}/LICENSE +0 -0
  94. {eodag-3.0.0b3.dist-info → eodag-3.1.0.dist-info}/top_level.txt +0 -0
@@ -0,0 +1,95 @@
1
+ # -*- coding: utf-8 -*-
2
+ # Copyright 2021, CS GROUP - France, http://www.c-s.fr
3
+ #
4
+ # This file is part of EODAG project
5
+ # https://www.github.com/CS-SI/EODAG
6
+ #
7
+ # Licensed under the Apache License, Version 2.0 (the "License");
8
+ # you may not use this file except in compliance with the License.
9
+ # You may obtain a copy of the License at
10
+ #
11
+ # http://www.apache.org/licenses/LICENSE-2.0
12
+ #
13
+ # Unless required by applicable law or agreed to in writing, software
14
+ # distributed under the License is distributed on an "AS IS" BASIS,
15
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16
+ # See the License for the specific language governing permissions and
17
+ # limitations under the License.
18
+ from __future__ import annotations
19
+
20
+ import re
21
+ from typing import TYPE_CHECKING
22
+
23
+ from eodag.api.product.drivers.base import AssetPatterns, DatasetDriver
24
+
25
+ if TYPE_CHECKING:
26
+ from eodag.api.product._product import EOProduct
27
+
28
+
29
+ class Sentinel2Driver(DatasetDriver):
30
+ """Driver for Sentinel2 products"""
31
+
32
+ #: Band keys associated with their default Ground Sampling Distance (GSD)
33
+ BANDS_DEFAULT_GSD = {
34
+ "10M": ("B02", "B03", "B04", "B08", "TCI"),
35
+ "20M": ("B05", "B06", "B07", "B11", "B12", "B8A"),
36
+ "60M": ("B01", "B09", "B10"),
37
+ }
38
+
39
+ #: list of patterns to match asset keys and roles
40
+ ASSET_KEYS_PATTERNS_ROLES: list[AssetPatterns] = [
41
+ # masks
42
+ {
43
+ "pattern": re.compile(r"^.*?(MSK_[^/\\]+)\.(?:jp2|tiff?)$", re.IGNORECASE),
44
+ "roles": ["data-mask"],
45
+ },
46
+ # visual
47
+ {
48
+ "pattern": re.compile(
49
+ r"^.*?(TCI)(_[0-9]+m)?\.(?:jp2|tiff?)$", re.IGNORECASE
50
+ ),
51
+ "roles": ["visual"],
52
+ },
53
+ # bands
54
+ {
55
+ "pattern": re.compile(
56
+ r"^.*?([A-Z]+[0-9]*[A-Z]?)(_[0-9]+m)?\.(?:jp2|tiff?)$", re.IGNORECASE
57
+ ),
58
+ "roles": ["data"],
59
+ },
60
+ # metadata
61
+ {
62
+ "pattern": re.compile(
63
+ r"^(?:.*[/\\])?([^/\\]+)(\.xml|\.xsd|\.safe|\.json)$", re.IGNORECASE
64
+ ),
65
+ "roles": ["metadata"],
66
+ },
67
+ # thumbnail
68
+ {
69
+ "pattern": re.compile(
70
+ r"^(?:.*[/\\])?(thumbnail)(\.jpe?g|\.png)$", re.IGNORECASE
71
+ ),
72
+ "roles": ["thumbnail"],
73
+ },
74
+ # quicklook
75
+ {
76
+ "pattern": re.compile(
77
+ r"^(?:.*[/\\])?[^/\\]+(-ql|preview|quick-?look)(\.jpe?g|\.png)$",
78
+ re.IGNORECASE,
79
+ ),
80
+ "roles": ["overview"],
81
+ },
82
+ # default
83
+ {"pattern": re.compile(r"^(?:.*[/\\])?([^/\\]+)$"), "roles": ["auxiliary"]},
84
+ ]
85
+
86
+ def _normalize_key(self, key: str, eo_product: EOProduct) -> str:
87
+ upper_key = key.upper()
88
+ # check if key matched any normalized
89
+ for res in self.BANDS_DEFAULT_GSD:
90
+ if res in upper_key:
91
+ for norm_key in self.BANDS_DEFAULT_GSD[res]:
92
+ if norm_key in upper_key:
93
+ return norm_key
94
+
95
+ return super()._normalize_key(key, eo_product)
@@ -23,24 +23,14 @@ import logging
23
23
  import re
24
24
  from datetime import datetime, timedelta
25
25
  from string import Formatter
26
- from typing import (
27
- TYPE_CHECKING,
28
- Any,
29
- AnyStr,
30
- Callable,
31
- Dict,
32
- Iterator,
33
- List,
34
- Optional,
35
- Tuple,
36
- Union,
37
- cast,
38
- )
26
+ from typing import TYPE_CHECKING, Any, AnyStr, Callable, Iterator, Optional, Union, cast
39
27
 
40
28
  import geojson
41
29
  import orjson
42
30
  import pyproj
31
+ import shapely
43
32
  from dateutil.parser import isoparse
33
+ from dateutil.relativedelta import relativedelta
44
34
  from dateutil.tz import UTC, tzutc
45
35
  from jsonpath_ng.jsonpath import Child, JSONPath
46
36
  from lxml import etree
@@ -86,8 +76,8 @@ DEFAULT_GEOMETRY = "POLYGON((180 -90, 180 90, -180 90, -180 -90, 180 -90))"
86
76
 
87
77
 
88
78
  def get_metadata_path(
89
- map_value: Union[str, List[str]],
90
- ) -> Tuple[Union[List[str], None], str]:
79
+ map_value: Union[str, list[str]],
80
+ ) -> tuple[Union[list[str], None], str]:
91
81
  """Return the jsonpath or xpath to the value of a EO product metadata in a provider
92
82
  search result.
93
83
 
@@ -135,12 +125,12 @@ def get_metadata_path(
135
125
  return None, path
136
126
 
137
127
 
138
- def get_metadata_path_value(map_value: Union[str, List[str]]) -> str:
128
+ def get_metadata_path_value(map_value: Union[str, list[str]]) -> str:
139
129
  """Get raw metadata path without converter"""
140
130
  return map_value[1] if isinstance(map_value, list) else map_value
141
131
 
142
132
 
143
- def get_search_param(map_value: List[str]) -> str:
133
+ def get_search_param(map_value: list[str]) -> str:
144
134
  """See :func:`~eodag.api.product.metadata_mapping.get_metadata_path`
145
135
 
146
136
  :param map_value: The value originating from the definition of `metadata_mapping`
@@ -152,7 +142,7 @@ def get_search_param(map_value: List[str]) -> str:
152
142
 
153
143
 
154
144
  def format_metadata(search_param: str, *args: Any, **kwargs: Any) -> str:
155
- """Format a string of form {<field_name>#<conversion_function>}
145
+ """Format a string of form ``{<field_name>#<conversion_function>}``
156
146
 
157
147
  The currently understood converters are:
158
148
  - ``datetime_to_timestamp_milliseconds``: converts a utc date string to a timestamp in
@@ -178,6 +168,8 @@ def format_metadata(search_param: str, *args: Any, **kwargs: Any) -> str:
178
168
  - ``recursive_sub_str``: recursively substitue in the structure (e.g. dict)
179
169
  values matching a regex
180
170
  - ``slice_str``: slice a string (equivalent to s[start, end, step])
171
+ - ``to_lower``: Convert a string to lowercase
172
+ - ``to_upper``: Convert a string to uppercase
181
173
  - ``fake_l2a_title_from_l1c``: used to generate SAFE format metadata for data from AWS
182
174
  - ``s2msil2a_title_to_aws_productinfo``: used to generate SAFE format metadata for data from AWS
183
175
  - ``split_cop_dem_id``: get the bbox by splitting the product id
@@ -331,7 +323,7 @@ def format_metadata(search_param: str, *args: Any, **kwargs: Any) -> str:
331
323
  return wkt_value
332
324
 
333
325
  @staticmethod
334
- def convert_to_bounds_lists(input_geom: BaseGeometry) -> List[List[float]]:
326
+ def convert_to_bounds_lists(input_geom: BaseGeometry) -> list[list[float]]:
335
327
  if isinstance(input_geom, MultiPolygon):
336
328
  geoms = [geom for geom in input_geom.geoms]
337
329
  # sort with larger one at first (stac-browser only plots first one)
@@ -341,7 +333,7 @@ def format_metadata(search_param: str, *args: Any, **kwargs: Any) -> str:
341
333
  return [list(input_geom.bounds[0:4])]
342
334
 
343
335
  @staticmethod
344
- def convert_to_bounds(input_geom_unformatted: Any) -> List[float]:
336
+ def convert_to_bounds(input_geom_unformatted: Any) -> list[float]:
345
337
  input_geom = get_geometry_from_various(geometry=input_geom_unformatted)
346
338
  if isinstance(input_geom, MultiPolygon):
347
339
  geoms = [geom for geom in input_geom.geoms]
@@ -352,16 +344,18 @@ def format_metadata(search_param: str, *args: Any, **kwargs: Any) -> str:
352
344
  max_lon = -180
353
345
  max_lat = -90
354
346
  for geom in geoms:
355
- min_lon = min(min_lon, geom.bound[0])
356
- min_lat = min(min_lat, geom.bound[1])
357
- max_lon = max(max_lon, geom.bound[2])
358
- max_lat = max(max_lat, geom.bound[3])
347
+ min_lon = min(min_lon, geom.bounds[0])
348
+ min_lat = min(min_lat, geom.bounds[1])
349
+ max_lon = max(max_lon, geom.bounds[2])
350
+ max_lat = max(max_lat, geom.bounds[3])
359
351
  return [min_lon, min_lat, max_lon, max_lat]
360
352
  else:
361
353
  return list(input_geom.bounds[0:4])
362
354
 
363
355
  @staticmethod
364
- def convert_to_nwse_bounds(input_geom: BaseGeometry) -> List[float]:
356
+ def convert_to_nwse_bounds(input_geom: BaseGeometry) -> list[float]:
357
+ if isinstance(input_geom, str):
358
+ input_geom = shapely.wkt.loads(input_geom)
365
359
  return list(input_geom.bounds[-1:] + input_geom.bounds[:-1])
366
360
 
367
361
  @staticmethod
@@ -373,8 +367,8 @@ def format_metadata(search_param: str, *args: Any, **kwargs: Any) -> str:
373
367
  )
374
368
 
375
369
  @staticmethod
376
- def convert_to_geojson(string: str) -> str:
377
- return geojson.dumps(string)
370
+ def convert_to_geojson(value: Any) -> str:
371
+ return geojson.dumps(value)
378
372
 
379
373
  @staticmethod
380
374
  def convert_from_ewkt(ewkt_string: str) -> Union[BaseGeometry, str]:
@@ -443,7 +437,7 @@ def format_metadata(search_param: str, *args: Any, **kwargs: Any) -> str:
443
437
  else:
444
438
  yield e
445
439
 
446
- polygons_list: List[Polygon] = []
440
+ polygons_list: list[Polygon] = []
447
441
  for elem in flatten_elements(georss[0]):
448
442
  coords_list = elem.text.split()
449
443
  polygon_args = [
@@ -468,7 +462,7 @@ def format_metadata(search_param: str, *args: Any, **kwargs: Any) -> str:
468
462
  @staticmethod
469
463
  def convert_to_longitude_latitude(
470
464
  input_geom_unformatted: Any,
471
- ) -> Dict[str, float]:
465
+ ) -> dict[str, float]:
472
466
  bounds = MetadataFormatter.convert_to_bounds(input_geom_unformatted)
473
467
  lon = (bounds[0] + bounds[2]) / 2
474
468
  lat = (bounds[1] + bounds[3]) / 2
@@ -502,14 +496,21 @@ def format_metadata(search_param: str, *args: Any, **kwargs: Any) -> str:
502
496
  return NOT_AVAILABLE
503
497
 
504
498
  @staticmethod
505
- def convert_replace_str(string: str, args: str) -> str:
499
+ def convert_replace_str(value: Any, args: str) -> str:
500
+ if isinstance(value, dict):
501
+ value = MetadataFormatter.convert_to_geojson(value)
502
+ elif not isinstance(value, str):
503
+ raise TypeError(
504
+ f"convert_replace_str expects a string or a dict (apply to_geojson). Got {type(value)}"
505
+ )
506
+
506
507
  old, new = ast.literal_eval(args)
507
- return re.sub(old, new, string)
508
+ return re.sub(old, new, value)
508
509
 
509
510
  @staticmethod
510
511
  def convert_recursive_sub_str(
511
- input_obj: Union[Dict[Any, Any], List[Any]], args: str
512
- ) -> Union[Dict[Any, Any], List[Any]]:
512
+ input_obj: Union[dict[Any, Any], list[Any]], args: str
513
+ ) -> Union[dict[Any, Any], list[Any]]:
513
514
  old, new = ast.literal_eval(args)
514
515
  return items_recursive_apply(
515
516
  input_obj,
@@ -519,8 +520,8 @@ def format_metadata(search_param: str, *args: Any, **kwargs: Any) -> str:
519
520
 
520
521
  @staticmethod
521
522
  def convert_dict_update(
522
- input_dict: Dict[Any, Any], args: str
523
- ) -> Dict[Any, Any]:
523
+ input_dict: dict[Any, Any], args: str
524
+ ) -> dict[Any, Any]:
524
525
  """Converts"""
525
526
  new_items_list = ast.literal_eval(args)
526
527
 
@@ -530,8 +531,8 @@ def format_metadata(search_param: str, *args: Any, **kwargs: Any) -> str:
530
531
 
531
532
  @staticmethod
532
533
  def convert_dict_filter(
533
- input_dict: Dict[Any, Any], jsonpath_filter_str: str
534
- ) -> Dict[Any, Any]:
534
+ input_dict: dict[Any, Any], jsonpath_filter_str: str
535
+ ) -> dict[Any, Any]:
535
536
  """Fitlers dict items using jsonpath"""
536
537
 
537
538
  jsonpath_filter = string_to_jsonpath(jsonpath_filter_str, force=True)
@@ -557,6 +558,16 @@ def format_metadata(search_param: str, *args: Any, **kwargs: Any) -> str:
557
558
  ]
558
559
  return string[cmin:cmax:cstep]
559
560
 
561
+ @staticmethod
562
+ def convert_to_lower(string: str) -> str:
563
+ """Convert a string to lowercase."""
564
+ return string.lower()
565
+
566
+ @staticmethod
567
+ def convert_to_upper(string: str) -> str:
568
+ """Convert a string to uppercase."""
569
+ return string.upper()
570
+
560
571
  @staticmethod
561
572
  def convert_fake_l2a_title_from_l1c(string: str) -> str:
562
573
  id_regex = re.compile(
@@ -600,8 +611,8 @@ def format_metadata(search_param: str, *args: Any, **kwargs: Any) -> str:
600
611
  return NOT_AVAILABLE
601
612
 
602
613
  @staticmethod
603
- def convert_split_id_into_s1_params(product_id: str) -> Dict[str, str]:
604
- parts: List[str] = re.split(r"_(?!_)", product_id)
614
+ def convert_split_id_into_s1_params(product_id: str) -> dict[str, str]:
615
+ parts: list[str] = re.split(r"_(?!_)", product_id)
605
616
  if len(parts) < 9:
606
617
  logger.error(
607
618
  "id %s does not match expected Sentinel-1 id format", product_id
@@ -635,8 +646,8 @@ def format_metadata(search_param: str, *args: Any, **kwargs: Any) -> str:
635
646
  return params
636
647
 
637
648
  @staticmethod
638
- def convert_split_id_into_s3_params(product_id: str) -> Dict[str, str]:
639
- parts: List[str] = re.split(r"_(?!_)", product_id)
649
+ def convert_split_id_into_s3_params(product_id: str) -> dict[str, str]:
650
+ parts: list[str] = re.split(r"_(?!_)", product_id)
640
651
  params = {"productType": product_id[4:15]}
641
652
  dates = re.findall("[0-9]{8}T[0-9]{6}", product_id)
642
653
  start_date = datetime.strptime(dates[0], "%Y%m%dT%H%M%S") - timedelta(
@@ -652,8 +663,8 @@ def format_metadata(search_param: str, *args: Any, **kwargs: Any) -> str:
652
663
  return params
653
664
 
654
665
  @staticmethod
655
- def convert_split_id_into_s5p_params(product_id: str) -> Dict[str, str]:
656
- parts: List[str] = re.split(r"_(?!_)", product_id)
666
+ def convert_split_id_into_s5p_params(product_id: str) -> dict[str, str]:
667
+ parts: list[str] = re.split(r"_(?!_)", product_id)
657
668
  params = {
658
669
  "productType": product_id[9:19],
659
670
  "processingMode": parts[1],
@@ -670,7 +681,7 @@ def format_metadata(search_param: str, *args: Any, **kwargs: Any) -> str:
670
681
  return params
671
682
 
672
683
  @staticmethod
673
- def convert_split_cop_dem_id(product_id: str) -> List[int]:
684
+ def convert_split_cop_dem_id(product_id: str) -> list[int]:
674
685
  parts = product_id.split("_")
675
686
  lattitude = parts[3]
676
687
  longitude = parts[5]
@@ -709,7 +720,7 @@ def format_metadata(search_param: str, *args: Any, **kwargs: Any) -> str:
709
720
  @staticmethod
710
721
  def convert_to_datetime_dict(
711
722
  date: str, format: str
712
- ) -> Dict[str, Union[List[str], str]]:
723
+ ) -> dict[str, Union[list[str], str]]:
713
724
  """Convert a date (str) to a dictionary where values are in the format given in argument
714
725
 
715
726
  date == "2021-04-21T18:27:19.123Z" and format == "list" => {
@@ -761,7 +772,7 @@ def format_metadata(search_param: str, *args: Any, **kwargs: Any) -> str:
761
772
  @staticmethod
762
773
  def convert_interval_to_datetime_dict(
763
774
  date: str, separator: str = "/"
764
- ) -> Dict[str, List[str]]:
775
+ ) -> dict[str, list[str]]:
765
776
  """Convert a date interval ('/' separated str) to a dictionary where values are lists
766
777
 
767
778
  date == "2021-04-21/2021-04-22" => {
@@ -801,7 +812,7 @@ def format_metadata(search_param: str, *args: Any, **kwargs: Any) -> str:
801
812
  }
802
813
 
803
814
  @staticmethod
804
- def convert_get_ecmwf_time(date: str) -> List[str]:
815
+ def convert_get_ecmwf_time(date: str) -> list[str]:
805
816
  """Get the time of a date (str) in the ECMWF format (["HH:00"])
806
817
 
807
818
  "2021-04-21T18:27:19.123Z" => ["18:00"]
@@ -831,9 +842,9 @@ def format_metadata(search_param: str, *args: Any, **kwargs: Any) -> str:
831
842
  def convert_get_hydrological_year(date: str):
832
843
  utc_date = MetadataFormatter.convert_to_iso_utc_datetime(date)
833
844
  date_object = datetime.strptime(utc_date, "%Y-%m-%dT%H:%M:%S.%fZ")
834
- date_object_second_year = date_object + timedelta(days=365)
845
+ date_object_second_year = date_object + relativedelta(years=1)
835
846
  return [
836
- f'{date_object.strftime("%Y")}_{date_object_second_year.strftime("%y")}'
847
+ f"{date_object.strftime('%Y')}_{date_object_second_year.strftime('%y')}"
837
848
  ]
838
849
 
839
850
  @staticmethod
@@ -845,8 +856,8 @@ def format_metadata(search_param: str, *args: Any, **kwargs: Any) -> str:
845
856
 
846
857
  @staticmethod
847
858
  def convert_assets_list_to_dict(
848
- assets_list: List[Dict[str, str]], asset_name_key: str = "title"
849
- ) -> Dict[str, Dict[str, str]]:
859
+ assets_list: list[dict[str, str]], asset_name_key: str = "title"
860
+ ) -> dict[str, dict[str, str]]:
850
861
  """Convert a list of assets to a dictionary where keys represent
851
862
  name of assets and are found among values of asset dictionaries.
852
863
 
@@ -873,8 +884,8 @@ def format_metadata(search_param: str, *args: Any, **kwargs: Any) -> str:
873
884
  "asset3": {"href": "qux", "title": "qux-title", "name": "asset3"},
874
885
  }
875
886
  """
876
- asset_names: List[str] = []
877
- assets_dict: Dict[str, Dict[str, str]] = {}
887
+ asset_names: list[str] = []
888
+ assets_dict: dict[str, dict[str, str]] = {}
878
889
 
879
890
  for asset in assets_list:
880
891
  asset_name = asset[asset_name_key]
@@ -883,7 +894,7 @@ def format_metadata(search_param: str, *args: Any, **kwargs: Any) -> str:
883
894
 
884
895
  # we only keep the equivalent of the path basename in the case where the
885
896
  # asset name has a path pattern and this basename is only found once
886
- immutable_asset_indexes: List[int] = []
897
+ immutable_asset_indexes: list[int] = []
887
898
  for i, asset_name in enumerate(asset_names):
888
899
  if i in immutable_asset_indexes:
889
900
  continue
@@ -901,20 +912,18 @@ def format_metadata(search_param: str, *args: Any, **kwargs: Any) -> str:
901
912
  return assets_dict
902
913
 
903
914
  # if stac extension colon separator `:` is in search params, parse it to prevent issues with vformat
904
- if re.search(r"{[a-zA-Z0-9_-]*:[a-zA-Z0-9_-]*}", search_param):
905
- search_param = re.sub(
906
- r"{([a-zA-Z0-9_-]*):([a-zA-Z0-9_-]*)}", r"{\1_COLON_\2}", search_param
907
- )
915
+ if re.search(r"{[\w-]*:[\w#-]*}", search_param):
916
+ search_param = re.sub(r"{([\w-]*):([\w#-]*)}", r"{\1_COLON_\2}", search_param)
908
917
  kwargs = {k.replace(":", "_COLON_"): v for k, v in kwargs.items()}
909
918
 
910
919
  return MetadataFormatter().vformat(search_param, args, kwargs)
911
920
 
912
921
 
913
922
  def properties_from_json(
914
- json: Dict[str, Any],
915
- mapping: Dict[str, Any],
916
- discovery_config: Optional[Dict[str, Any]] = None,
917
- ) -> Dict[str, Any]:
923
+ json: dict[str, Any],
924
+ mapping: dict[str, Any],
925
+ discovery_config: Optional[dict[str, Any]] = None,
926
+ ) -> dict[str, Any]:
918
927
  """Extract properties from a provider json result.
919
928
 
920
929
  :param json: The representation of a provider result as a json object
@@ -927,7 +936,7 @@ def properties_from_json(
927
936
  `discovery_path` (String representation of jsonpath)
928
937
  :returns: The metadata of the :class:`~eodag.api.product._product.EOProduct`
929
938
  """
930
- properties: Dict[str, Any] = {}
939
+ properties: dict[str, Any] = {}
931
940
  templates = {}
932
941
  used_jsonpaths = []
933
942
  for metadata, value in mapping.items():
@@ -974,10 +983,24 @@ def properties_from_json(
974
983
  if re.search(r"({[^{}:]+})+", conversion_or_none):
975
984
  conversion_or_none = conversion_or_none.format(**properties)
976
985
 
977
- properties[metadata] = format_metadata(
978
- "{%s%s%s}" % (metadata, SEP, conversion_or_none),
979
- **{metadata: extracted_value},
980
- )
986
+ if extracted_value == NOT_AVAILABLE:
987
+ # try if value can be formatted even if it is not available
988
+ try:
989
+ properties[metadata] = format_metadata(
990
+ "{%s%s%s}" % (metadata, SEP, conversion_or_none),
991
+ **{metadata: extracted_value},
992
+ )
993
+ except ValueError:
994
+ logger.debug(
995
+ f"{metadata}: {extracted_value} could not be formatted with {conversion_or_none}"
996
+ )
997
+ continue
998
+ else:
999
+ # in this case formatting should work, otherwise something is wrong in the mapping
1000
+ properties[metadata] = format_metadata(
1001
+ "{%s%s%s}" % (metadata, SEP, conversion_or_none),
1002
+ **{metadata: extracted_value},
1003
+ )
981
1004
  # properties as python objects when possible (format_metadata returns only strings)
982
1005
  try:
983
1006
  properties[metadata] = ast.literal_eval(properties[metadata])
@@ -1057,8 +1080,8 @@ def properties_from_xml(
1057
1080
  xml_as_text: AnyStr,
1058
1081
  mapping: Any,
1059
1082
  empty_ns_prefix: str = "ns",
1060
- discovery_config: Optional[Dict[str, Any]] = None,
1061
- ) -> Dict[str, Any]:
1083
+ discovery_config: Optional[dict[str, Any]] = None,
1084
+ ) -> dict[str, Any]:
1062
1085
  """Extract properties from a provider xml result.
1063
1086
 
1064
1087
  :param xml_as_text: The representation of a provider result as xml
@@ -1076,7 +1099,7 @@ def properties_from_xml(
1076
1099
  `discovery_path` (String representation of xpath)
1077
1100
  :returns: the metadata of the :class:`~eodag.api.product._product.EOProduct`
1078
1101
  """
1079
- properties: Dict[str, Any] = {}
1102
+ properties: dict[str, Any] = {}
1080
1103
  templates = {}
1081
1104
  used_xpaths = []
1082
1105
  root = etree.XML(xml_as_text)
@@ -1204,11 +1227,11 @@ def properties_from_xml(
1204
1227
 
1205
1228
 
1206
1229
  def mtd_cfg_as_conversion_and_querypath(
1207
- src_dict: Dict[str, Any],
1208
- dest_dict: Dict[str, Any] = {},
1230
+ src_dict: dict[str, Any],
1231
+ dest_dict: dict[str, Any] = {},
1209
1232
  result_type: str = "json",
1210
- ) -> Dict[str, Any]:
1211
- """Metadata configuration dictionary to querypath with conversion dictionnary
1233
+ ) -> dict[str, Any]:
1234
+ """Metadata configuration dictionary to querypath with conversion dictionary
1212
1235
  Transform every src_dict value from jsonpath_str to tuple `(conversion, jsonpath_object)`
1213
1236
  or from xpath_str to tuple `(conversion, xpath_str)`
1214
1237
 
@@ -1255,8 +1278,8 @@ def mtd_cfg_as_conversion_and_querypath(
1255
1278
 
1256
1279
 
1257
1280
  def format_query_params(
1258
- product_type: str, config: PluginConfig, query_dict: Dict[str, Any]
1259
- ) -> Dict[str, Any]:
1281
+ product_type: str, config: PluginConfig, query_dict: dict[str, Any]
1282
+ ) -> dict[str, Any]:
1260
1283
  """format the search parameters to query parameters"""
1261
1284
  if "raise_errors" in query_dict.keys():
1262
1285
  del query_dict["raise_errors"]
@@ -1268,7 +1291,7 @@ def format_query_params(
1268
1291
  **config.products.get(product_type, {}).get("metadata_mapping", {}),
1269
1292
  )
1270
1293
 
1271
- query_params: Dict[str, Any] = {}
1294
+ query_params: dict[str, Any] = {}
1272
1295
  # Get all the search parameters that are recognised as queryables by the
1273
1296
  # provider (they appear in the queryables dictionary)
1274
1297
  queryables = _get_queryables(query_dict, config, product_type_metadata_mapping)
@@ -1298,8 +1321,8 @@ def format_query_params(
1298
1321
  query_params[eodag_search_key] = formatted_query_param
1299
1322
  else:
1300
1323
  provider_search_key, provider_value = parts
1301
- query_params.setdefault(provider_search_key, []).append(
1302
- format_metadata(provider_value, product_type, **query_dict)
1324
+ query_params[provider_search_key] = format_metadata(
1325
+ provider_value, product_type, **query_dict
1303
1326
  )
1304
1327
  else:
1305
1328
  query_params[provider_search_key] = user_input
@@ -1358,10 +1381,10 @@ def _resolve_hashes(formatted_query_param: str) -> str:
1358
1381
 
1359
1382
 
1360
1383
  def _format_free_text_search(
1361
- config: PluginConfig, metadata_mapping: Dict[str, Any], **kwargs: Any
1362
- ) -> Dict[str, Any]:
1384
+ config: PluginConfig, metadata_mapping: dict[str, Any], **kwargs: Any
1385
+ ) -> dict[str, Any]:
1363
1386
  """Build the free text search parameter using the search parameters"""
1364
- query_params: Dict[str, Any] = {}
1387
+ query_params: dict[str, Any] = {}
1365
1388
  if not getattr(config, "free_text_search_operations", None):
1366
1389
  return query_params
1367
1390
  for param, operations_config in config.free_text_search_operations.items():
@@ -1400,13 +1423,13 @@ def _format_free_text_search(
1400
1423
 
1401
1424
 
1402
1425
  def _get_queryables(
1403
- search_params: Dict[str, Any],
1426
+ search_params: dict[str, Any],
1404
1427
  config: PluginConfig,
1405
- metadata_mapping: Dict[str, Any],
1406
- ) -> Dict[str, Any]:
1428
+ metadata_mapping: dict[str, Any],
1429
+ ) -> dict[str, Any]:
1407
1430
  """Retrieve the metadata mappings that are query-able"""
1408
1431
  logger.debug("Retrieving queryable metadata from metadata_mapping")
1409
- queryables: Dict[str, Any] = {}
1432
+ queryables: dict[str, Any] = {}
1410
1433
  for eodag_search_key, user_input in search_params.items():
1411
1434
  if user_input is not None:
1412
1435
  md_mapping = metadata_mapping.get(eodag_search_key, (None, NOT_MAPPED))
@@ -1453,7 +1476,7 @@ def _get_queryables(
1453
1476
 
1454
1477
 
1455
1478
  def get_queryable_from_provider(
1456
- provider_queryable: str, metadata_mapping: Dict[str, Union[str, List[str]]]
1479
+ provider_queryable: str, metadata_mapping: dict[str, Union[str, list[str]]]
1457
1480
  ) -> Optional[str]:
1458
1481
  """Get EODAG configured queryable parameter from provider queryable parameter
1459
1482
 
@@ -1461,15 +1484,27 @@ def get_queryable_from_provider(
1461
1484
  :param metadata_mapping: metadata-mapping configuration
1462
1485
  :returns: EODAG configured queryable parameter or None
1463
1486
  """
1464
- pattern = rf"\b{provider_queryable}\b"
1487
+ pattern = rf"\"{provider_queryable}\""
1488
+ # if 1:1 mapping exists privilege this one instead of other mapping
1489
+ # e.g. provider queryable = year -> use year and not date in which year also appears
1490
+ mapping_values = [
1491
+ v[0] if isinstance(v, list) else "" for v in metadata_mapping.values()
1492
+ ]
1493
+ if provider_queryable in mapping_values:
1494
+ ind = mapping_values.index(provider_queryable)
1495
+ return Queryables.get_queryable_from_alias(list(metadata_mapping.keys())[ind])
1465
1496
  for param, param_conf in metadata_mapping.items():
1466
- if isinstance(param_conf, list) and re.search(pattern, param_conf[0]):
1497
+ if (
1498
+ isinstance(param_conf, list)
1499
+ and param_conf[0]
1500
+ and re.search(pattern, param_conf[0])
1501
+ ):
1467
1502
  return Queryables.get_queryable_from_alias(param)
1468
1503
  return None
1469
1504
 
1470
1505
 
1471
1506
  def get_provider_queryable_path(
1472
- queryable: str, metadata_mapping: Dict[str, Union[str, List[str]]]
1507
+ queryable: str, metadata_mapping: dict[str, Union[str, list[str]]]
1473
1508
  ) -> Optional[str]:
1474
1509
  """Get EODAG configured queryable path from its parameter
1475
1510
 
@@ -1486,10 +1521,11 @@ def get_provider_queryable_path(
1486
1521
 
1487
1522
  def get_provider_queryable_key(
1488
1523
  eodag_key: str,
1489
- provider_queryables: Dict[str, Any],
1490
- metadata_mapping: Dict[str, Union[List[Any], str]],
1524
+ provider_queryables: dict[str, Any],
1525
+ metadata_mapping: dict[str, Union[list[Any], str]],
1491
1526
  ) -> str:
1492
- """finds the provider queryable corresponding to the given eodag key based on the metadata mapping
1527
+ """Finds the provider queryable corresponding to the given eodag key based on the metadata mapping
1528
+
1493
1529
  :param eodag_key: key in eodag
1494
1530
  :param provider_queryables: queryables returned from the provider
1495
1531
  :param metadata_mapping: metadata mapping from which the keys are retrieved