eodag 3.0.1__py3-none-any.whl → 3.1.0b1__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (44) hide show
  1. eodag/api/core.py +116 -86
  2. eodag/api/product/_assets.py +6 -6
  3. eodag/api/product/_product.py +18 -18
  4. eodag/api/product/metadata_mapping.py +39 -11
  5. eodag/cli.py +22 -1
  6. eodag/config.py +14 -14
  7. eodag/plugins/apis/ecmwf.py +37 -14
  8. eodag/plugins/apis/usgs.py +5 -5
  9. eodag/plugins/authentication/openid_connect.py +2 -2
  10. eodag/plugins/authentication/token.py +37 -6
  11. eodag/plugins/crunch/filter_property.py +2 -3
  12. eodag/plugins/download/aws.py +11 -12
  13. eodag/plugins/download/base.py +30 -39
  14. eodag/plugins/download/creodias_s3.py +29 -0
  15. eodag/plugins/download/http.py +144 -152
  16. eodag/plugins/download/s3rest.py +5 -7
  17. eodag/plugins/search/base.py +73 -25
  18. eodag/plugins/search/build_search_result.py +1047 -310
  19. eodag/plugins/search/creodias_s3.py +25 -19
  20. eodag/plugins/search/data_request_search.py +1 -1
  21. eodag/plugins/search/qssearch.py +51 -139
  22. eodag/resources/ext_product_types.json +1 -1
  23. eodag/resources/product_types.yml +391 -32
  24. eodag/resources/providers.yml +678 -1744
  25. eodag/rest/core.py +92 -62
  26. eodag/rest/server.py +31 -4
  27. eodag/rest/types/eodag_search.py +6 -0
  28. eodag/rest/types/queryables.py +5 -6
  29. eodag/rest/utils/__init__.py +3 -0
  30. eodag/types/__init__.py +56 -15
  31. eodag/types/download_args.py +2 -2
  32. eodag/types/queryables.py +180 -72
  33. eodag/types/whoosh.py +126 -0
  34. eodag/utils/__init__.py +71 -10
  35. eodag/utils/exceptions.py +27 -20
  36. eodag/utils/repr.py +65 -6
  37. eodag/utils/requests.py +11 -11
  38. {eodag-3.0.1.dist-info → eodag-3.1.0b1.dist-info}/METADATA +76 -76
  39. {eodag-3.0.1.dist-info → eodag-3.1.0b1.dist-info}/RECORD +43 -44
  40. {eodag-3.0.1.dist-info → eodag-3.1.0b1.dist-info}/WHEEL +1 -1
  41. {eodag-3.0.1.dist-info → eodag-3.1.0b1.dist-info}/entry_points.txt +3 -2
  42. eodag/utils/constraints.py +0 -244
  43. {eodag-3.0.1.dist-info → eodag-3.1.0b1.dist-info}/LICENSE +0 -0
  44. {eodag-3.0.1.dist-info → eodag-3.1.0b1.dist-info}/top_level.txt +0 -0
eodag/api/core.py CHANGED
@@ -24,27 +24,15 @@ import re
24
24
  import shutil
25
25
  import tempfile
26
26
  from operator import itemgetter
27
- from typing import (
28
- TYPE_CHECKING,
29
- Annotated,
30
- Any,
31
- Dict,
32
- Iterator,
33
- List,
34
- Optional,
35
- Set,
36
- Tuple,
37
- Union,
38
- )
27
+ from typing import TYPE_CHECKING, Any, Dict, Iterator, List, Optional, Tuple, Union
39
28
 
40
29
  import geojson
41
30
  import pkg_resources
42
31
  import yaml.parser
43
32
  from pkg_resources import resource_filename
44
- from pydantic.fields import FieldInfo
45
33
  from whoosh import analysis, fields
46
34
  from whoosh.fields import Schema
47
- from whoosh.index import create_in, exists_in, open_dir
35
+ from whoosh.index import exists_in, open_dir
48
36
  from whoosh.qparser import QueryParser
49
37
 
50
38
  from eodag.api.product.metadata_mapping import (
@@ -69,11 +57,11 @@ from eodag.config import (
69
57
  )
70
58
  from eodag.plugins.manager import PluginManager
71
59
  from eodag.plugins.search import PreparedSearch
72
- from eodag.plugins.search.build_search_result import BuildPostSearchResult
60
+ from eodag.plugins.search.build_search_result import MeteoblueSearch
73
61
  from eodag.plugins.search.qssearch import PostJsonSearch
74
62
  from eodag.types import model_fields_to_annotated
75
- from eodag.types.queryables import CommonQueryables
76
- from eodag.types.whoosh import EODAGQueryParser
63
+ from eodag.types.queryables import CommonQueryables, QueryablesDict
64
+ from eodag.types.whoosh import EODAGQueryParser, create_in
77
65
  from eodag.utils import (
78
66
  DEFAULT_DOWNLOAD_TIMEOUT,
79
67
  DEFAULT_DOWNLOAD_WAIT,
@@ -84,7 +72,6 @@ from eodag.utils import (
84
72
  HTTP_REQ_TIMEOUT,
85
73
  MockResponse,
86
74
  _deprecated,
87
- copy_deepcopy,
88
75
  get_geometry_from_various,
89
76
  makedirs,
90
77
  obj_md5sum,
@@ -93,6 +80,7 @@ from eodag.utils import (
93
80
  uri_to_path,
94
81
  )
95
82
  from eodag.utils.exceptions import (
83
+ AuthenticationError,
96
84
  EodagError,
97
85
  NoMatchingProductType,
98
86
  PluginImplementationError,
@@ -219,9 +207,10 @@ class EODataAccessGateway:
219
207
  "eodag",
220
208
  os.path.join("resources", "locations_conf_template.yml"),
221
209
  )
222
- with open(locations_conf_template) as infile, open(
223
- locations_conf_path, "w"
224
- ) as outfile:
210
+ with (
211
+ open(locations_conf_template) as infile,
212
+ open(locations_conf_path, "w") as outfile,
213
+ ):
225
214
  # The template contains paths in the form of:
226
215
  # /path/to/locations/file.shp
227
216
  path_template = "/path/to/locations/"
@@ -317,13 +306,18 @@ class EODataAccessGateway:
317
306
  product_type, **{"md5": self.product_types_config_md5}
318
307
  )
319
308
  # add to index
320
- ix_writer.add_document(
321
- **{
322
- k: v
323
- for k, v in versioned_product_type.items()
324
- if k in product_types_schema.names()
325
- }
326
- )
309
+ try:
310
+ ix_writer.add_document(
311
+ **{
312
+ k: v
313
+ for k, v in versioned_product_type.items()
314
+ if k in product_types_schema.names()
315
+ }
316
+ )
317
+ except TypeError as e:
318
+ logger.error(
319
+ f"Cannot write product type {product_type['ID']} into index. e={e} product_type={product_type}"
320
+ )
327
321
  ix_writer.commit()
328
322
 
329
323
  def set_preferred_provider(self, provider: str) -> None:
@@ -1773,12 +1767,12 @@ class EODataAccessGateway:
1773
1767
  for plugin in self._plugins_manager.get_search_plugins(
1774
1768
  product_type=product_type, provider=provider
1775
1769
  ):
1776
- # exclude BuildPostSearchResult plugins from search fallback for unknow product_type
1770
+ # exclude MeteoblueSearch plugins from search fallback for unknown product_type
1777
1771
  if (
1778
1772
  provider != plugin.provider
1779
1773
  and preferred_provider != plugin.provider
1780
1774
  and product_type not in self.product_types_config
1781
- and isinstance(plugin, BuildPostSearchResult)
1775
+ and isinstance(plugin, MeteoblueSearch)
1782
1776
  ):
1783
1777
  continue
1784
1778
  search_plugins.append(plugin)
@@ -1801,27 +1795,7 @@ class EODataAccessGateway:
1801
1795
  # Add product_types_config to plugin config. This dict contains product
1802
1796
  # type metadata that will also be stored in each product's properties.
1803
1797
  for search_plugin in search_plugins:
1804
- try:
1805
- search_plugin.config.product_type_config = dict(
1806
- [
1807
- p
1808
- for p in self.list_product_types(
1809
- search_plugin.provider, fetch_providers=False
1810
- )
1811
- if p["_id"] == product_type
1812
- ][0],
1813
- **{"productType": product_type},
1814
- )
1815
- # If the product isn't in the catalog, it's a generic product type.
1816
- except IndexError:
1817
- # Construct the GENERIC_PRODUCT_TYPE metadata
1818
- search_plugin.config.product_type_config = dict(
1819
- ID=GENERIC_PRODUCT_TYPE,
1820
- **self.product_types_config[GENERIC_PRODUCT_TYPE],
1821
- productType=product_type,
1822
- )
1823
- # Remove the ID since this is equal to productType.
1824
- search_plugin.config.product_type_config.pop("ID", None)
1798
+ self._attach_product_type_config(search_plugin, product_type)
1825
1799
 
1826
1800
  return search_plugins, kwargs
1827
1801
 
@@ -2038,8 +2012,8 @@ class EODataAccessGateway:
2038
2012
  search_result: SearchResult,
2039
2013
  downloaded_callback: Optional[DownloadedCallback] = None,
2040
2014
  progress_callback: Optional[ProgressCallback] = None,
2041
- wait: int = DEFAULT_DOWNLOAD_WAIT,
2042
- timeout: int = DEFAULT_DOWNLOAD_TIMEOUT,
2015
+ wait: float = DEFAULT_DOWNLOAD_WAIT,
2016
+ timeout: float = DEFAULT_DOWNLOAD_TIMEOUT,
2043
2017
  **kwargs: Unpack[DownloadConf],
2044
2018
  ) -> List[str]:
2045
2019
  """Download all products resulting from a search.
@@ -2201,8 +2175,8 @@ class EODataAccessGateway:
2201
2175
  self,
2202
2176
  product: EOProduct,
2203
2177
  progress_callback: Optional[ProgressCallback] = None,
2204
- wait: int = DEFAULT_DOWNLOAD_WAIT,
2205
- timeout: int = DEFAULT_DOWNLOAD_TIMEOUT,
2178
+ wait: float = DEFAULT_DOWNLOAD_WAIT,
2179
+ timeout: float = DEFAULT_DOWNLOAD_TIMEOUT,
2206
2180
  **kwargs: Unpack[DownloadConf],
2207
2181
  ) -> str:
2208
2182
  """Download a single product.
@@ -2280,67 +2254,96 @@ class EODataAccessGateway:
2280
2254
  return self._plugins_manager.get_crunch_plugin(name, **plugin_conf)
2281
2255
 
2282
2256
  def list_queryables(
2283
- self, provider: Optional[str] = None, **kwargs: Any
2284
- ) -> Dict[str, Annotated[Any, FieldInfo]]:
2257
+ self,
2258
+ provider: Optional[str] = None,
2259
+ fetch_providers: bool = True,
2260
+ **kwargs: Any,
2261
+ ) -> QueryablesDict:
2285
2262
  """Fetch the queryable properties for a given product type and/or provider.
2286
2263
 
2287
2264
  :param provider: (optional) The provider.
2265
+ :param fetch_providers: If new product types should be fetched from the providers; default: True
2288
2266
  :param kwargs: additional filters for queryables (`productType` or other search
2289
2267
  arguments)
2290
2268
 
2291
2269
  :raises UnsupportedProductType: If the specified product type is not available for the
2292
2270
  provider.
2293
2271
 
2294
- :returns: A dict containing the EODAG queryable properties, associating
2295
- parameters to their annotated type
2272
+ :returns: A :class:`~eodag.api.product.queryables.QuerybalesDict` containing the EODAG queryable
2273
+ properties, associating parameters to their annotated type, and a additional_properties attribute
2296
2274
  """
2275
+ # only fetch providers if product type is not found
2297
2276
  available_product_types = [
2298
2277
  pt["ID"]
2299
2278
  for pt in self.list_product_types(provider=provider, fetch_providers=False)
2300
2279
  ]
2301
- product_type = kwargs.get("productType")
2280
+ product_type: Optional[str] = kwargs.get("productType")
2281
+ pt_alias: Optional[str] = product_type
2302
2282
 
2303
2283
  if product_type:
2284
+ if product_type not in available_product_types:
2285
+ if fetch_providers:
2286
+ # fetch providers and try again
2287
+ available_product_types = [
2288
+ pt["ID"]
2289
+ for pt in self.list_product_types(
2290
+ provider=provider, fetch_providers=True
2291
+ )
2292
+ ]
2293
+ raise UnsupportedProductType(f"{product_type} is not available.")
2304
2294
  try:
2305
2295
  kwargs["productType"] = product_type = self.get_product_type_from_alias(
2306
2296
  product_type
2307
2297
  )
2308
2298
  except NoMatchingProductType as e:
2309
- raise UnsupportedProductType(f"{product_type} is not available") from e
2310
-
2311
- if product_type and product_type not in available_product_types:
2312
- self.fetch_product_types_list()
2299
+ raise UnsupportedProductType(f"{product_type} is not available.") from e
2313
2300
 
2314
2301
  if not provider and not product_type:
2315
- return model_fields_to_annotated(CommonQueryables.model_fields)
2302
+ return QueryablesDict(
2303
+ additional_properties=True,
2304
+ **model_fields_to_annotated(CommonQueryables.model_fields),
2305
+ )
2316
2306
 
2317
- providers_queryables: Dict[str, Dict[str, Annotated[Any, FieldInfo]]] = {}
2307
+ queryables: QueryablesDict = QueryablesDict(
2308
+ additional_properties=True, additional_information="", **{}
2309
+ )
2318
2310
 
2319
2311
  for plugin in self._plugins_manager.get_search_plugins(product_type, provider):
2312
+ # attach product type config
2313
+ product_type_configs = {}
2314
+ if product_type:
2315
+ self._attach_product_type_config(plugin, product_type)
2316
+ product_type_configs[product_type] = plugin.config.product_type_config
2317
+ else:
2318
+ for pt in available_product_types:
2319
+ self._attach_product_type_config(plugin, pt)
2320
+ product_type_configs[pt] = plugin.config.product_type_config
2321
+
2322
+ # authenticate if required
2320
2323
  if getattr(plugin.config, "need_auth", False) and (
2321
2324
  auth := self._plugins_manager.get_auth_plugin(plugin)
2322
2325
  ):
2323
- plugin.auth = auth.authenticate()
2324
- providers_queryables[plugin.provider] = plugin.list_queryables(
2325
- filters=kwargs, product_type=product_type
2326
+ try:
2327
+ plugin.auth = auth.authenticate()
2328
+ except AuthenticationError:
2329
+ logger.debug(
2330
+ "queryables from provider %s could not be fetched due to an authentication error",
2331
+ plugin.provider,
2332
+ )
2333
+ plugin_queryables = plugin.list_queryables(
2334
+ kwargs,
2335
+ available_product_types,
2336
+ product_type_configs,
2337
+ product_type,
2338
+ pt_alias,
2326
2339
  )
2327
-
2328
- queryable_keys: Set[str] = set.intersection( # type: ignore
2329
- *[set(q.keys()) for q in providers_queryables.values()]
2330
- )
2331
- queryables = {
2332
- k: v
2333
- for k, v in list(providers_queryables.values())[0].items()
2334
- if k in queryable_keys
2335
- }
2336
-
2337
- # always keep at least CommonQueryables
2338
- common_queryables = copy_deepcopy(CommonQueryables.model_fields)
2339
- for key, queryable in common_queryables.items():
2340
- if key in kwargs:
2341
- queryable.default = kwargs[key]
2342
-
2343
- queryables.update(model_fields_to_annotated(common_queryables))
2340
+ queryables.update(plugin_queryables)
2341
+ if not plugin_queryables.additional_properties:
2342
+ queryables.additional_properties = False
2343
+ if plugin_queryables.additional_information:
2344
+ queryables.additional_information += (
2345
+ f"{provider}: {plugin_queryables.additional_information}"
2346
+ )
2344
2347
 
2345
2348
  return queryables
2346
2349
 
@@ -2375,3 +2378,30 @@ class EODataAccessGateway:
2375
2378
  ],
2376
2379
  }
2377
2380
  return sortables
2381
+
2382
+ def _attach_product_type_config(self, plugin: Search, product_type: str) -> None:
2383
+ """
2384
+ Attach product_types_config to plugin config. This dict contains product
2385
+ type metadata that will also be stored in each product's properties.
2386
+ """
2387
+ try:
2388
+ plugin.config.product_type_config = dict(
2389
+ [
2390
+ p
2391
+ for p in self.list_product_types(
2392
+ plugin.provider, fetch_providers=False
2393
+ )
2394
+ if p["_id"] == product_type
2395
+ ][0],
2396
+ **{"productType": product_type},
2397
+ )
2398
+ # If the product isn't in the catalog, it's a generic product type.
2399
+ except IndexError:
2400
+ # Construct the GENERIC_PRODUCT_TYPE metadata
2401
+ plugin.config.product_type_config = dict(
2402
+ ID=GENERIC_PRODUCT_TYPE,
2403
+ **self.product_types_config[GENERIC_PRODUCT_TYPE],
2404
+ productType=product_type,
2405
+ )
2406
+ # Remove the ID since this is equal to productType.
2407
+ plugin.config.product_type_config.pop("ID", None)
@@ -98,12 +98,12 @@ class AssetsDict(UserDict):
98
98
  <details><summary style='color: grey;'>
99
99
  <span style='color: black'>'{k}'</span>:&ensp;
100
100
  {{
101
- {"'roles': '<span style='color: black'>"+str(v['roles'])+"</span>',&ensp;"
102
- if v.get("roles") else ""}
103
- {"'type': '"+str(v['type'])+"',&ensp;"
104
- if v.get("type") else ""}
105
- {"'title': '<span style='color: black'>"+str(v['title'])+"</span>',&ensp;"
106
- if v.get("title") else ""}
101
+ {"'roles': '<span style='color: black'>" + str(v['roles']) + "</span>',&ensp;"
102
+ if v.get("roles") else ""}
103
+ {"'type': '" + str(v['type']) + "',&ensp;"
104
+ if v.get("type") else ""}
105
+ {"'title': '<span style='color: black'>" + str(v['title']) + "</span>',&ensp;"
106
+ if v.get("title") else ""}
107
107
  ...
108
108
  }}
109
109
  </summary>
@@ -116,6 +116,7 @@ class EOProduct:
116
116
  properties: Dict[str, Any]
117
117
  product_type: Optional[str]
118
118
  location: str
119
+ filename: str
119
120
  remote_location: str
120
121
  search_kwargs: Any
121
122
  geometry: BaseGeometry
@@ -281,8 +282,8 @@ class EOProduct:
281
282
  def download(
282
283
  self,
283
284
  progress_callback: Optional[ProgressCallback] = None,
284
- wait: int = DEFAULT_DOWNLOAD_WAIT,
285
- timeout: int = DEFAULT_DOWNLOAD_TIMEOUT,
285
+ wait: float = DEFAULT_DOWNLOAD_WAIT,
286
+ timeout: float = DEFAULT_DOWNLOAD_TIMEOUT,
286
287
  **kwargs: Unpack[DownloadConf],
287
288
  ) -> str:
288
289
  """Download the EO product using the provided download plugin and the
@@ -525,22 +526,21 @@ class EOProduct:
525
526
  <tr style='background-color: transparent;'>
526
527
  <td style='text-align: left; vertical-align: top;'>
527
528
  {dict_to_html_table({
528
- "provider": self.provider,
529
- "product_type": self.product_type,
530
- "properties[&quot;id&quot;]": self.properties.get('id', None),
531
- "properties[&quot;startTimeFromAscendingNode&quot;]": self.properties.get(
532
- 'startTimeFromAscendingNode', None
533
- ),
534
- "properties[&quot;completionTimeFromAscendingNode&quot;]": self.properties.get(
535
- 'completionTimeFromAscendingNode', None
536
- ),
537
- }, brackets=False)}
538
- <details><summary style='color: grey; margin-top: 10px;'>properties:&ensp;({
539
- len(self.properties)
540
- })</summary>{dict_to_html_table(self.properties, depth=1)}</details>
541
- <details><summary style='color: grey; margin-top: 10px;'>assets:&ensp;({
542
- len(self.assets)
543
- })</summary>{self.assets._repr_html_(embeded=True)}</details>
529
+ "provider": self.provider,
530
+ "product_type": self.product_type,
531
+ "properties[&quot;id&quot;]": self.properties.get('id', None),
532
+ "properties[&quot;startTimeFromAscendingNode&quot;]": self.properties.get(
533
+ 'startTimeFromAscendingNode', None
534
+ ),
535
+ "properties[&quot;completionTimeFromAscendingNode&quot;]": self.properties.get(
536
+ 'completionTimeFromAscendingNode', None
537
+ ),
538
+ }, brackets=False)}
539
+ <details><summary style='color: grey; margin-top: 10px;'>properties:&ensp;({len(
540
+ self.properties)})</summary>{
541
+ dict_to_html_table(self.properties, depth=1)}</details>
542
+ <details><summary style='color: grey; margin-top: 10px;'>assets:&ensp;({len(
543
+ self.assets)})</summary>{self.assets._repr_html_(embeded=True)}</details>
544
544
  </td>
545
545
  <td {geom_style} title='geometry'>geometry<br />{self.geometry._repr_svg_()}</td>
546
546
  <td {thumbnail_style} title='properties[&quot;thumbnail&quot;]'>{thumbnail_html}</td>
@@ -40,6 +40,7 @@ from typing import (
40
40
  import geojson
41
41
  import orjson
42
42
  import pyproj
43
+ import shapely
43
44
  from dateutil.parser import isoparse
44
45
  from dateutil.relativedelta import relativedelta
45
46
  from dateutil.tz import UTC, tzutc
@@ -179,6 +180,8 @@ def format_metadata(search_param: str, *args: Any, **kwargs: Any) -> str:
179
180
  - ``recursive_sub_str``: recursively substitue in the structure (e.g. dict)
180
181
  values matching a regex
181
182
  - ``slice_str``: slice a string (equivalent to s[start, end, step])
183
+ - ``to_lower``: Convert a string to lowercase
184
+ - ``to_upper``: Convert a string to uppercase
182
185
  - ``fake_l2a_title_from_l1c``: used to generate SAFE format metadata for data from AWS
183
186
  - ``s2msil2a_title_to_aws_productinfo``: used to generate SAFE format metadata for data from AWS
184
187
  - ``split_cop_dem_id``: get the bbox by splitting the product id
@@ -363,6 +366,8 @@ def format_metadata(search_param: str, *args: Any, **kwargs: Any) -> str:
363
366
 
364
367
  @staticmethod
365
368
  def convert_to_nwse_bounds(input_geom: BaseGeometry) -> List[float]:
369
+ if isinstance(input_geom, str):
370
+ input_geom = shapely.wkt.loads(input_geom)
366
371
  return list(input_geom.bounds[-1:] + input_geom.bounds[:-1])
367
372
 
368
373
  @staticmethod
@@ -558,6 +563,16 @@ def format_metadata(search_param: str, *args: Any, **kwargs: Any) -> str:
558
563
  ]
559
564
  return string[cmin:cmax:cstep]
560
565
 
566
+ @staticmethod
567
+ def convert_to_lower(string: str) -> str:
568
+ """Convert a string to lowercase."""
569
+ return string.lower()
570
+
571
+ @staticmethod
572
+ def convert_to_upper(string: str) -> str:
573
+ """Convert a string to uppercase."""
574
+ return string.upper()
575
+
561
576
  @staticmethod
562
577
  def convert_fake_l2a_title_from_l1c(string: str) -> str:
563
578
  id_regex = re.compile(
@@ -834,7 +849,7 @@ def format_metadata(search_param: str, *args: Any, **kwargs: Any) -> str:
834
849
  date_object = datetime.strptime(utc_date, "%Y-%m-%dT%H:%M:%S.%fZ")
835
850
  date_object_second_year = date_object + relativedelta(years=1)
836
851
  return [
837
- f'{date_object.strftime("%Y")}_{date_object_second_year.strftime("%y")}'
852
+ f"{date_object.strftime('%Y')}_{date_object_second_year.strftime('%y')}"
838
853
  ]
839
854
 
840
855
  @staticmethod
@@ -902,10 +917,8 @@ def format_metadata(search_param: str, *args: Any, **kwargs: Any) -> str:
902
917
  return assets_dict
903
918
 
904
919
  # if stac extension colon separator `:` is in search params, parse it to prevent issues with vformat
905
- if re.search(r"{[a-zA-Z0-9_-]*:[a-zA-Z0-9_-]*}", search_param):
906
- search_param = re.sub(
907
- r"{([a-zA-Z0-9_-]*):([a-zA-Z0-9_-]*)}", r"{\1_COLON_\2}", search_param
908
- )
920
+ if re.search(r"{[\w-]*:[\w#-]*}", search_param):
921
+ search_param = re.sub(r"{([\w-]*):([\w#-]*)}", r"{\1_COLON_\2}", search_param)
909
922
  kwargs = {k.replace(":", "_COLON_"): v for k, v in kwargs.items()}
910
923
 
911
924
  return MetadataFormatter().vformat(search_param, args, kwargs)
@@ -975,10 +988,24 @@ def properties_from_json(
975
988
  if re.search(r"({[^{}:]+})+", conversion_or_none):
976
989
  conversion_or_none = conversion_or_none.format(**properties)
977
990
 
978
- properties[metadata] = format_metadata(
979
- "{%s%s%s}" % (metadata, SEP, conversion_or_none),
980
- **{metadata: extracted_value},
981
- )
991
+ if extracted_value == NOT_AVAILABLE:
992
+ # try if value can be formatted even if it is not available
993
+ try:
994
+ properties[metadata] = format_metadata(
995
+ "{%s%s%s}" % (metadata, SEP, conversion_or_none),
996
+ **{metadata: extracted_value},
997
+ )
998
+ except ValueError:
999
+ logger.debug(
1000
+ f"{metadata}: {extracted_value} could not be formatted with {conversion_or_none}"
1001
+ )
1002
+ continue
1003
+ else:
1004
+ # in this case formatting should work, otherwise something is wrong in the mapping
1005
+ properties[metadata] = format_metadata(
1006
+ "{%s%s%s}" % (metadata, SEP, conversion_or_none),
1007
+ **{metadata: extracted_value},
1008
+ )
982
1009
  # properties as python objects when possible (format_metadata returns only strings)
983
1010
  try:
984
1011
  properties[metadata] = ast.literal_eval(properties[metadata])
@@ -1462,7 +1489,7 @@ def get_queryable_from_provider(
1462
1489
  :param metadata_mapping: metadata-mapping configuration
1463
1490
  :returns: EODAG configured queryable parameter or None
1464
1491
  """
1465
- pattern = rf"\b{provider_queryable}\b"
1492
+ pattern = rf"\"{provider_queryable}\""
1466
1493
  # if 1:1 mapping exists privilege this one instead of other mapping
1467
1494
  # e.g. provider queryable = year -> use year and not date in which year also appears
1468
1495
  mapping_values = [
@@ -1498,7 +1525,8 @@ def get_provider_queryable_key(
1498
1525
  provider_queryables: Dict[str, Any],
1499
1526
  metadata_mapping: Dict[str, Union[List[Any], str]],
1500
1527
  ) -> str:
1501
- """finds the provider queryable corresponding to the given eodag key based on the metadata mapping
1528
+ """Finds the provider queryable corresponding to the given eodag key based on the metadata mapping
1529
+
1502
1530
  :param eodag_key: key in eodag
1503
1531
  :param provider_queryables: queryables returned from the provider
1504
1532
  :param metadata_mapping: metadata mapping from which the keys are retrieved
eodag/cli.py CHANGED
@@ -57,6 +57,11 @@ from eodag.utils import DEFAULT_ITEMS_PER_PAGE, DEFAULT_PAGE, parse_qs
57
57
  from eodag.utils.exceptions import NoMatchingProductType, UnsupportedProvider
58
58
  from eodag.utils.logging import setup_logging
59
59
 
60
+ try:
61
+ from eodag.rest.utils import LIVENESS_PROBE_PATH
62
+ except ImportError:
63
+ pass
64
+
60
65
  if TYPE_CHECKING:
61
66
  from click import Context
62
67
 
@@ -70,6 +75,18 @@ CRUNCHERS = [
70
75
  ]
71
76
 
72
77
 
78
+ class LivenessFilter:
79
+ """
80
+ Filter out requests to the liveness probe endpoint
81
+ """
82
+
83
+ def filter(self, record):
84
+ """
85
+ Filter method required by the Python logging API.
86
+ """
87
+ return LIVENESS_PROBE_PATH not in record.getMessage()
88
+
89
+
73
90
  class MutuallyExclusiveOption(click.Option):
74
91
  """Mutually Exclusive Options for Click
75
92
  from https://gist.github.com/jacobtolar/fb80d5552a9a9dfc32b12a829fa21c0c
@@ -679,7 +696,9 @@ def serve_rest(
679
696
  try:
680
697
  pid = os.fork()
681
698
  except OSError as e:
682
- raise Exception("%s [%d]" % (e.strerror, e.errno))
699
+ raise Exception(
700
+ "%s [%d]" % (e.strerror, e.errno) if e.errno is not None else e.strerror
701
+ )
683
702
 
684
703
  if pid == 0:
685
704
  os.setsid()
@@ -691,8 +710,10 @@ def serve_rest(
691
710
 
692
711
  logging_config = uvicorn.config.LOGGING_CONFIG
693
712
  uvicorn_fmt = "%(asctime)-15s %(name)-32s [%(levelname)-8s] %(message)s"
713
+ logging_config["filters"] = {"liveness": {"()": LivenessFilter}}
694
714
  logging_config["formatters"]["access"]["fmt"] = uvicorn_fmt
695
715
  logging_config["formatters"]["default"]["fmt"] = uvicorn_fmt
716
+ logging_config["loggers"]["uvicorn.access"]["filters"] = ["liveness"]
696
717
 
697
718
  eodag_formatter = logging.Formatter(
698
719
  "%(asctime)-15s %(name)-32s [%(levelname)-8s] (tid=%(thread)d) %(message)s"
eodag/config.py CHANGED
@@ -287,8 +287,10 @@ class PluginConfig(yaml.YAMLObject):
287
287
  #: Mapping for product type metadata (e.g. ``abstract``, ``licence``) which can be parsed from the provider
288
288
  #: result
289
289
  generic_product_type_parsable_metadata: Dict[str, str]
290
- #: Mapping for product type properties which can be parsed from the result that are not product type metadata
290
+ #: Mapping for product type properties which can be parsed from the result and are not product type metadata
291
291
  generic_product_type_parsable_properties: Dict[str, str]
292
+ #: Mapping for product type properties which cannot be parsed from the result and are not product type metadata
293
+ generic_product_type_unparsable_properties: Dict[str, str]
292
294
  #: URL to fetch data for a single collection
293
295
  single_collection_fetch_url: str
294
296
  #: Query string to be added to the fetch_url to filter for a collection
@@ -307,6 +309,10 @@ class PluginConfig(yaml.YAMLObject):
307
309
  result_type: str
308
310
  #: JsonPath to retrieve the queryables from the provider result
309
311
  results_entry: str
312
+ #: :class:`~eodag.plugins.search.base.Search` URL of the constraint file used to build queryables
313
+ constraints_url: str
314
+ #: :class:`~eodag.plugins.search.base.Search` Key in the json result where the constraints can be found
315
+ constraints_entry: str
310
316
 
311
317
  class OrderOnResponse(TypedDict):
312
318
  """Configuration for order on-response during download"""
@@ -434,17 +440,6 @@ class PluginConfig(yaml.YAMLObject):
434
440
  discover_queryables: PluginConfig.DiscoverQueryables
435
441
  #: :class:`~eodag.plugins.search.base.Search` The mapping between eodag metadata and the plugin specific metadata
436
442
  metadata_mapping: Dict[str, Union[str, List[str]]]
437
- #: :class:`~eodag.plugins.search.base.Search` URL of the constraint file used to build queryables
438
- constraints_file_url: str
439
- #: :class:`~eodag.plugins.search.base.Search`
440
- #: Key which is used in the eodag configuration to map the eodag product type to the provider product type
441
- constraints_file_dataset_key: str
442
- #: :class:`~eodag.plugins.search.base.Search` Key in the json result where the constraints can be found
443
- constraints_entry: str
444
- #: :class:`~eodag.plugins.search.base.Search`
445
- #: Whether only a provider result containing constraints_entry is accepted as valid and used to create constraints
446
- #: or not
447
- stop_without_constraints_entry_key: bool
448
443
  #: :class:`~eodag.plugins.search.base.Search` Parameters to remove from queryables
449
444
  remove_from_queryables: List[str]
450
445
  #: :class:`~eodag.plugins.search.base.Search` Parameters to be passed as is in the search url query string
@@ -477,15 +472,17 @@ class PluginConfig(yaml.YAMLObject):
477
472
  #: :class:`~eodag.plugins.search.static_stac_search.StaticStacSearch`
478
473
  #: Maximum number of connections for concurrent HTTP requests
479
474
  max_connections: int
480
- #: :class:`~eodag.plugins.search.build_search_result.BuildSearchResult`
475
+ #: :class:`~eodag.plugins.search.build_search_result.ECMWFSearch`
481
476
  #: Whether end date should be excluded from search request or not
482
477
  end_date_excluded: bool
483
- #: :class:`~eodag.plugins.search.build_search_result.BuildSearchResult`
478
+ #: :class:`~eodag.plugins.search.build_search_result.ECMWFSearch`
484
479
  #: List of parameters used to parse metadata but that must not be included to the query
485
480
  remove_from_query: List[str]
486
481
  #: :class:`~eodag.plugins.search.csw.CSWSearch`
487
482
  #: OGC Catalogue Service version
488
483
  version: str
484
+ #: :class:`~eodag.plugins.apis.ecmwf.EcmwfApi` url of the authentication endpoint
485
+ auth_endpoint: str
489
486
 
490
487
  # download ---------------------------------------------------------------------------------------------------------
491
488
  #: :class:`~eodag.plugins.download.base.Download` Default endpoint url
@@ -539,6 +536,9 @@ class PluginConfig(yaml.YAMLObject):
539
536
  #: Dictionary containing all keys/value pairs that should be added to the headers
540
537
  headers: Dict[str, str]
541
538
  #: :class:`~eodag.plugins.authentication.base.Authentication`
539
+ #: Dictionary containing all keys/value pairs that should be added to the headers for token retrieve only
540
+ retrieve_headers: Dict[str, str]
541
+ #: :class:`~eodag.plugins.authentication.base.Authentication`
542
542
  #: The key pointing to the token in the response from the token server
543
543
  token_key: str
544
544
  #: :class:`~eodag.plugins.authentication.base.Authentication`