eodag 3.0.0b3__py3-none-any.whl → 3.1.0b1__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (77) hide show
  1. eodag/api/core.py +292 -198
  2. eodag/api/product/_assets.py +6 -6
  3. eodag/api/product/_product.py +18 -18
  4. eodag/api/product/metadata_mapping.py +51 -14
  5. eodag/api/search_result.py +29 -3
  6. eodag/cli.py +57 -20
  7. eodag/config.py +413 -117
  8. eodag/plugins/apis/base.py +10 -4
  9. eodag/plugins/apis/ecmwf.py +49 -16
  10. eodag/plugins/apis/usgs.py +30 -7
  11. eodag/plugins/authentication/aws_auth.py +14 -5
  12. eodag/plugins/authentication/base.py +10 -1
  13. eodag/plugins/authentication/generic.py +14 -3
  14. eodag/plugins/authentication/header.py +12 -4
  15. eodag/plugins/authentication/keycloak.py +41 -22
  16. eodag/plugins/authentication/oauth.py +11 -1
  17. eodag/plugins/authentication/openid_connect.py +178 -163
  18. eodag/plugins/authentication/qsauth.py +12 -4
  19. eodag/plugins/authentication/sas_auth.py +19 -2
  20. eodag/plugins/authentication/token.py +93 -15
  21. eodag/plugins/authentication/token_exchange.py +19 -19
  22. eodag/plugins/crunch/base.py +4 -1
  23. eodag/plugins/crunch/filter_date.py +5 -2
  24. eodag/plugins/crunch/filter_latest_intersect.py +5 -4
  25. eodag/plugins/crunch/filter_latest_tpl_name.py +1 -1
  26. eodag/plugins/crunch/filter_overlap.py +5 -7
  27. eodag/plugins/crunch/filter_property.py +6 -6
  28. eodag/plugins/download/aws.py +50 -34
  29. eodag/plugins/download/base.py +41 -50
  30. eodag/plugins/download/creodias_s3.py +40 -2
  31. eodag/plugins/download/http.py +221 -195
  32. eodag/plugins/download/s3rest.py +25 -25
  33. eodag/plugins/manager.py +168 -23
  34. eodag/plugins/search/base.py +106 -39
  35. eodag/plugins/search/build_search_result.py +1065 -324
  36. eodag/plugins/search/cop_marine.py +112 -29
  37. eodag/plugins/search/creodias_s3.py +45 -24
  38. eodag/plugins/search/csw.py +41 -1
  39. eodag/plugins/search/data_request_search.py +109 -9
  40. eodag/plugins/search/qssearch.py +549 -257
  41. eodag/plugins/search/static_stac_search.py +20 -21
  42. eodag/resources/ext_product_types.json +1 -1
  43. eodag/resources/product_types.yml +577 -87
  44. eodag/resources/providers.yml +1619 -2776
  45. eodag/resources/stac.yml +3 -163
  46. eodag/resources/user_conf_template.yml +112 -97
  47. eodag/rest/config.py +1 -2
  48. eodag/rest/constants.py +0 -1
  49. eodag/rest/core.py +138 -98
  50. eodag/rest/errors.py +181 -0
  51. eodag/rest/server.py +55 -329
  52. eodag/rest/stac.py +93 -544
  53. eodag/rest/types/eodag_search.py +19 -8
  54. eodag/rest/types/queryables.py +6 -8
  55. eodag/rest/types/stac_search.py +11 -2
  56. eodag/rest/utils/__init__.py +3 -0
  57. eodag/types/__init__.py +71 -18
  58. eodag/types/download_args.py +3 -3
  59. eodag/types/queryables.py +180 -73
  60. eodag/types/search_args.py +3 -3
  61. eodag/types/whoosh.py +126 -0
  62. eodag/utils/__init__.py +147 -66
  63. eodag/utils/exceptions.py +47 -26
  64. eodag/utils/logging.py +37 -77
  65. eodag/utils/repr.py +65 -6
  66. eodag/utils/requests.py +11 -13
  67. eodag/utils/stac_reader.py +1 -1
  68. {eodag-3.0.0b3.dist-info → eodag-3.1.0b1.dist-info}/METADATA +80 -81
  69. eodag-3.1.0b1.dist-info/RECORD +108 -0
  70. {eodag-3.0.0b3.dist-info → eodag-3.1.0b1.dist-info}/WHEEL +1 -1
  71. {eodag-3.0.0b3.dist-info → eodag-3.1.0b1.dist-info}/entry_points.txt +4 -2
  72. eodag/resources/constraints/climate-dt.json +0 -13
  73. eodag/resources/constraints/extremes-dt.json +0 -8
  74. eodag/utils/constraints.py +0 -244
  75. eodag-3.0.0b3.dist-info/RECORD +0 -110
  76. {eodag-3.0.0b3.dist-info → eodag-3.1.0b1.dist-info}/LICENSE +0 -0
  77. {eodag-3.0.0b3.dist-info → eodag-3.1.0b1.dist-info}/top_level.txt +0 -0
eodag/config.py CHANGED
@@ -22,6 +22,7 @@ import os
22
22
  import tempfile
23
23
  from inspect import isclass
24
24
  from typing import (
25
+ Annotated,
25
26
  Any,
26
27
  Dict,
27
28
  ItemsView,
@@ -44,14 +45,11 @@ import yaml.parser
44
45
  from annotated_types import Gt
45
46
  from jsonpath_ng import JSONPath
46
47
  from pkg_resources import resource_filename
47
- from requests.auth import AuthBase
48
- from typing_extensions import Doc
49
48
 
50
49
  from eodag.api.product.metadata_mapping import mtd_cfg_as_conversion_and_querypath
51
50
  from eodag.utils import (
52
51
  HTTP_REQ_TIMEOUT,
53
52
  USER_AGENT,
54
- Annotated,
55
53
  cached_yaml_load,
56
54
  cached_yaml_load_all,
57
55
  cast_scalar_value,
@@ -59,6 +57,7 @@ from eodag.utils import (
59
57
  dict_items_recursive_apply,
60
58
  merge_mappings,
61
59
  slugify,
60
+ sort_dict,
62
61
  string_to_jsonpath,
63
62
  update_nested_dict,
64
63
  uri_to_path,
@@ -70,6 +69,8 @@ logger = logging.getLogger("eodag.config")
70
69
  EXT_PRODUCT_TYPES_CONF_URI = (
71
70
  "https://cs-si.github.io/eodag/eodag/resources/ext_product_types.json"
72
71
  )
72
+ AUTH_TOPIC_KEYS = ("auth", "search_auth", "download_auth")
73
+ PLUGINS_TOPICS_KEYS = ("api", "search", "download") + AUTH_TOPIC_KEYS
73
74
 
74
75
 
75
76
  class SimpleYamlProxyConfig:
@@ -118,6 +119,8 @@ class ProviderConfig(yaml.YAMLObject):
118
119
  :param products: (optional) The products types supported by the provider
119
120
  :param download: (optional) The configuration of a plugin of type Download
120
121
  :param auth: (optional) The configuration of a plugin of type Authentication
122
+ :param search_auth: (optional) The configuration of a plugin of type Authentication for search
123
+ :param download_auth: (optional) The configuration of a plugin of type Authentication for download
121
124
  :param kwargs: Additional configuration variables for this provider
122
125
  """
123
126
 
@@ -132,6 +135,8 @@ class ProviderConfig(yaml.YAMLObject):
132
135
  products: Dict[str, Any]
133
136
  download: PluginConfig
134
137
  auth: PluginConfig
138
+ search_auth: PluginConfig
139
+ download_auth: PluginConfig
135
140
  product_types_fetched: bool # set in core.update_product_types_list
136
141
 
137
142
  yaml_loader = yaml.Loader
@@ -152,7 +157,7 @@ class ProviderConfig(yaml.YAMLObject):
152
157
  def from_mapping(cls, mapping: Dict[str, Any]) -> ProviderConfig:
153
158
  """Build a :class:`~eodag.config.ProviderConfig` from a mapping"""
154
159
  cls.validate(mapping)
155
- for key in ("api", "search", "download", "auth"):
160
+ for key in PLUGINS_TOPICS_KEYS:
156
161
  if key in mapping:
157
162
  mapping[key] = PluginConfig.from_mapping(mapping[key])
158
163
  c = cls()
@@ -167,11 +172,10 @@ class ProviderConfig(yaml.YAMLObject):
167
172
  """
168
173
  if "name" not in config_keys:
169
174
  raise ValidationError("Provider config must have name key")
170
- if not any(k in config_keys for k in ("api", "search", "download", "auth")):
175
+ if not any(k in config_keys for k in PLUGINS_TOPICS_KEYS):
171
176
  raise ValidationError("A provider must implement at least one plugin")
172
- if "api" in config_keys and any(
173
- k in config_keys for k in ("search", "download", "auth")
174
- ):
177
+ non_api_keys = [k for k in PLUGINS_TOPICS_KEYS if k != "api"]
178
+ if "api" in config_keys and any(k in config_keys for k in non_api_keys):
175
179
  raise ValidationError(
176
180
  "A provider implementing an Api plugin must not implement any other "
177
181
  "type of plugin"
@@ -189,11 +193,10 @@ class ProviderConfig(yaml.YAMLObject):
189
193
  {
190
194
  key: value
191
195
  for key, value in mapping.items()
192
- if key not in ("name", "api", "search", "download", "auth")
193
- and value is not None
196
+ if key not in PLUGINS_TOPICS_KEYS and value is not None
194
197
  },
195
198
  )
196
- for key in ("api", "search", "download", "auth"):
199
+ for key in PLUGINS_TOPICS_KEYS:
197
200
  current_value: Optional[Dict[str, Any]] = getattr(self, key, None)
198
201
  mapping_value = mapping.get(key, {})
199
202
  if current_value is not None:
@@ -203,48 +206,118 @@ class ProviderConfig(yaml.YAMLObject):
203
206
 
204
207
 
205
208
  class PluginConfig(yaml.YAMLObject):
206
- """Representation of a plugin config
209
+ """Representation of a plugin config.
207
210
 
208
- :param name: The name of the plugin class to use to instantiate the plugin object
209
- :param metadata_mapping: (optional) The mapping between eodag metadata and
210
- the plugin specific metadata
211
- :param free_params: (optional) Additional configuration parameters
211
+ This class variables describe available plugins configuration parameters.
212
212
  """
213
213
 
214
214
  class Pagination(TypedDict):
215
215
  """Search pagination configuration"""
216
216
 
217
+ #: The maximum number of items per page that the provider can handle
217
218
  max_items_per_page: int
219
+ #: Key path for the number of total items in the provider result
218
220
  total_items_nb_key_path: Union[str, JSONPath]
221
+ #: Key path for the next page URL
219
222
  next_page_url_key_path: Union[str, JSONPath]
223
+ #: Key path for the next page POST request query-object (body)
220
224
  next_page_query_obj_key_path: Union[str, JSONPath]
225
+ # TODO: change this typing to bool and adapt code to it
221
226
  next_page_merge_key_path: Union[str, JSONPath]
227
+ #: Template to add to :attr:`~eodag.config.PluginConfig.Pagination.next_page_url_tpl` to enable count in
228
+ #: search request
222
229
  count_tpl: str
230
+ #: The f-string template for pagination requests.
223
231
  next_page_url_tpl: str
232
+ #: The query-object for POST pagination requests.
224
233
  next_page_query_obj: str
234
+ #: The endpoint for counting the number of items satisfying a request
225
235
  count_endpoint: str
236
+ #: Index of the starting page
226
237
  start_page: int
227
238
 
228
239
  class Sort(TypedDict):
229
240
  """Configuration for sort during search"""
230
241
 
242
+ #: Default sort settings
231
243
  sort_by_default: List[Tuple[str, str]]
244
+ #: F-string template to add to :attr:`~eodag.config.PluginConfig.Pagination.next_page_url_tpl` to sort search
245
+ #: results
232
246
  sort_by_tpl: str
247
+ #: Mapping between eodag and provider query parameters used for sort
233
248
  sort_param_mapping: Dict[str, str]
249
+ #: Mapping between eodag and provider sort-order parameters
234
250
  sort_order_mapping: Dict[Literal["ascending", "descending"], str]
251
+ #: Maximum number of allowed sort parameters per request
235
252
  max_sort_params: Annotated[int, Gt(0)]
236
253
 
237
254
  class DiscoverMetadata(TypedDict):
238
- """Configuration for metadata discovery"""
255
+ """Configuration for metadata discovery (search result properties)"""
239
256
 
257
+ #: Whether metadata discovery is enabled or not
240
258
  auto_discovery: bool
259
+ #: Metadata regex pattern used for discovery in search result properties
241
260
  metadata_pattern: str
261
+ #: Configuration/template that will be used to query for a discovered parameter
242
262
  search_param: str
263
+ #: Path to the metadata in search result
243
264
  metadata_path: str
244
265
 
266
+ class DiscoverProductTypes(TypedDict, total=False):
267
+ """Configuration for product types discovery"""
268
+
269
+ #: URL from which the product types can be fetched
270
+ fetch_url: Optional[str]
271
+ #: HTTP method used to fetch product types
272
+ fetch_method: str
273
+ #: Request body to fetch product types using POST method
274
+ fetch_body: Dict[str, Any]
275
+ #: Maximum number of connections for concurrent HTTP requests
276
+ max_connections: int
277
+ #: The f-string template for pagination requests.
278
+ next_page_url_tpl: str
279
+ #: Index of the starting page for pagination requests.
280
+ start_page: int
281
+ #: Type of the provider result
282
+ result_type: str
283
+ #: JsonPath to the list of product types
284
+ results_entry: Union[JSONPath, str]
285
+ #: Mapping for the product type id
286
+ generic_product_type_id: str
287
+ #: Mapping for product type metadata (e.g. ``abstract``, ``licence``) which can be parsed from the provider
288
+ #: result
289
+ generic_product_type_parsable_metadata: Dict[str, str]
290
+ #: Mapping for product type properties which can be parsed from the result and are not product type metadata
291
+ generic_product_type_parsable_properties: Dict[str, str]
292
+ #: Mapping for product type properties which cannot be parsed from the result and are not product type metadata
293
+ generic_product_type_unparsable_properties: Dict[str, str]
294
+ #: URL to fetch data for a single collection
295
+ single_collection_fetch_url: str
296
+ #: Query string to be added to the fetch_url to filter for a collection
297
+ single_collection_fetch_qs: str
298
+ #: Mapping for product type metadata returned by the endpoint given in single_collection_fetch_url
299
+ single_product_type_parsable_metadata: Dict[str, str]
300
+
301
+ class DiscoverQueryables(TypedDict, total=False):
302
+ """Configuration for queryables discovery"""
303
+
304
+ #: URL to fetch the queryables valid for all product types
305
+ fetch_url: Optional[str]
306
+ #: URL to fetch the queryables for a specific product type
307
+ product_type_fetch_url: Optional[str]
308
+ #: Type of the result
309
+ result_type: str
310
+ #: JsonPath to retrieve the queryables from the provider result
311
+ results_entry: str
312
+ #: :class:`~eodag.plugins.search.base.Search` URL of the constraint file used to build queryables
313
+ constraints_url: str
314
+ #: :class:`~eodag.plugins.search.base.Search` Key in the json result where the constraints can be found
315
+ constraints_entry: str
316
+
245
317
  class OrderOnResponse(TypedDict):
246
318
  """Configuration for order on-response during download"""
247
319
 
320
+ #: Parameters metadata-mapping to apply to the order response
248
321
  metadata_mapping: Dict[str, Union[str, List[str]]]
249
322
 
250
323
  class OrderStatusSuccess(TypedDict):
@@ -255,136 +328,306 @@ class PluginConfig(yaml.YAMLObject):
255
328
  At least one is required
256
329
  """
257
330
 
258
- status: Annotated[str, Doc("Variable in the order status response json body")]
259
- message: Annotated[str, Doc("Variable in the order status response json body")]
260
- http_code: Annotated[int, Doc("HTTP code of the order status response")]
331
+ #: Success value for ``status``
332
+ status: str
333
+ #: Success value for ``message``
334
+ message: str
335
+ #: Success value for status response HTTP code
336
+ http_code: int
261
337
 
262
- class OrderStatusOrdered(TypedDict):
338
+ class OrderStatusOrdered(TypedDict, total=False):
263
339
  """
264
340
  Configuration to identify order status ordered during download
265
341
  """
266
342
 
267
- http_code: Annotated[int, Doc("HTTP code of the order status response")]
343
+ #: HTTP code of the order status response
344
+ http_code: int
268
345
 
269
- class OrderStatusRequest(TypedDict):
346
+ class OrderStatusRequest(TypedDict, total=False):
270
347
  """
271
348
  Order status request configuration
272
349
  """
273
350
 
274
- method: Annotated[str, Doc("Request HTTP method")]
275
- headers: Annotated[Dict[str, Any], Doc("Request hearders")]
351
+ #: Request HTTP method
352
+ method: str
353
+ #: Request hearders
354
+ headers: Dict[str, Any]
276
355
 
277
- class OrderStatusOnSuccess(TypedDict):
356
+ class OrderStatusOnSuccess(TypedDict, total=False):
278
357
  """Configuration for order status on-success during download"""
279
358
 
280
- need_search: Annotated[bool, Doc("If a new search is needed on success")]
359
+ #: Whether a new search is needed on success or not
360
+ need_search: bool
361
+ #: Return type of the success result
281
362
  result_type: str
363
+ #: Key in the success response that gives access to the result
282
364
  results_entry: str
365
+ #: Metadata-mapping to apply to the success status result
283
366
  metadata_mapping: Dict[str, Union[str, List[str]]]
284
367
 
285
- class OrderStatus(TypedDict):
368
+ class OrderStatus(TypedDict, total=False):
286
369
  """Configuration for order status during download"""
287
370
 
371
+ #: Order status request configuration
288
372
  request: PluginConfig.OrderStatusRequest
289
- metadata_mapping: Annotated[
290
- Dict[str, Union[str, List[str]]],
291
- Doc("Metadata-mapping used to parse order status response"),
292
- ]
373
+ #: Metadata-mapping used to parse order status response
374
+ metadata_mapping: Dict[str, Union[str, List[str]]]
375
+ #: Configuration to identify order status success during download
293
376
  success: PluginConfig.OrderStatusSuccess
294
- error: Annotated[
295
- Dict[str, Any],
296
- Doc("Part of the order status response that tells there is an error"),
297
- ]
377
+ #: Part of the order status response that tells there is an error
378
+ error: Dict[str, Any]
379
+ #: Configuration to identify order status ordered during download
298
380
  ordered: PluginConfig.OrderStatusOrdered
381
+ #: Configuration for order status on-success during download
299
382
  on_success: PluginConfig.OrderStatusOnSuccess
300
383
 
384
+ class MetadataPreMapping(TypedDict, total=False):
385
+ """Configuration which can be used to simplify further metadata extraction"""
386
+
387
+ #: JsonPath of the metadata entry
388
+ metadata_path: str
389
+ #: Key to get the metadata id
390
+ metadata_path_id: str
391
+ #: Key to get the metadata value
392
+ metadata_path_value: str
393
+
394
+ #: :class:`~eodag.plugins.base.PluginTopic` The name of the plugin class to use to instantiate the plugin object
301
395
  name: str
396
+ #: :class:`~eodag.plugins.base.PluginTopic` Plugin type
302
397
  type: str
303
-
304
- # search & api ---------------------------------------------------------------------
305
- priority: int # copied from ProviderConfig in PluginManager.get_search_plugins()
306
- products: Dict[
307
- str, Any
308
- ] # copied from ProviderConfig in PluginManager.get_search_plugins()
309
- product_type_config: Dict[str, Any] # set in core._prepare_search
310
- auth: Union[AuthBase, Dict[str, str]] # set in core._do_search
398
+ #: :class:`~eodag.plugins.base.PluginTopic` Whether the ssl certificates should be verified in the request or not
399
+ ssl_verify: bool
400
+ #: :class:`~eodag.plugins.base.PluginTopic` Default s3 bucket
401
+ s3_bucket: str
402
+ #: :class:`~eodag.plugins.base.PluginTopic` Authentication error codes
403
+ auth_error_code: Union[int, List[int]]
404
+ #: :class:`~eodag.plugins.base.PluginTopic` Time to wait until request timeout in seconds
405
+ timeout: float
406
+ #: :class:`~eodag.plugins.base.PluginTopic` :class:`urllib3.util.Retry` ``total`` parameter,
407
+ #: total number of retries to allow
408
+ retry_total: int
409
+ #: :class:`~eodag.plugins.base.PluginTopic` :class:`urllib3.util.Retry` ``backoff_factor`` parameter,
410
+ #: backoff factor to apply between attempts after the second try
411
+ retry_backoff_factor: int
412
+ #: :class:`~eodag.plugins.base.PluginTopic` :class:`urllib3.util.Retry` ``status_forcelist`` parameter,
413
+ #: list of integer HTTP status codes that we should force a retry on
414
+ retry_status_forcelist: List[int]
415
+
416
+ # search & api -----------------------------------------------------------------------------------------------------
417
+ # copied from ProviderConfig in PluginManager.get_search_plugins()
418
+ priority: int
419
+ # per product type metadata-mapping, set in core._prepare_search
420
+ product_type_config: Dict[str, Any]
421
+
422
+ #: :class:`~eodag.plugins.search.base.Search` Plugin API endpoint
311
423
  api_endpoint: str
424
+ #: :class:`~eodag.plugins.search.base.Search` Whether Search plugin needs authentification or not
312
425
  need_auth: bool
426
+ #: :class:`~eodag.plugins.search.base.Search` Return type of the provider result
313
427
  result_type: str
428
+ #: :class:`~eodag.plugins.search.base.Search`
429
+ #: Key in the provider search result that gives access to the result entries
314
430
  results_entry: str
431
+ #: :class:`~eodag.plugins.search.base.Search` Dict containing parameters for pagination
315
432
  pagination: PluginConfig.Pagination
433
+ #: :class:`~eodag.plugins.search.base.Search` Configuration for sorting the results
316
434
  sort: PluginConfig.Sort
317
- query_params_key: str
435
+ #: :class:`~eodag.plugins.search.base.Search` Configuration for the metadata auto-discovery
318
436
  discover_metadata: PluginConfig.DiscoverMetadata
319
- discover_product_types: Dict[str, Any]
320
- discover_queryables: Dict[str, Any]
437
+ #: :class:`~eodag.plugins.search.base.Search` Configuration for the product types auto-discovery
438
+ discover_product_types: PluginConfig.DiscoverProductTypes
439
+ #: :class:`~eodag.plugins.search.base.Search` Configuration for the queryables auto-discovery
440
+ discover_queryables: PluginConfig.DiscoverQueryables
441
+ #: :class:`~eodag.plugins.search.base.Search` The mapping between eodag metadata and the plugin specific metadata
321
442
  metadata_mapping: Dict[str, Union[str, List[str]]]
322
- free_params: Dict[Any, Any]
323
- constraints_file_url: str
443
+ #: :class:`~eodag.plugins.search.base.Search` Parameters to remove from queryables
324
444
  remove_from_queryables: List[str]
325
- free_text_search_operations: Dict[str, Any] # ODataV4Search
326
- metadata_pre_mapping: Dict[str, Any] # ODataV4Search
327
- data_request_url: str # DataRequestSearch
328
- status_url: str # DataRequestSearch
329
- result_url: str # DataRequestSearch
330
- search_definition: Dict[str, Any] # CSWSearch
331
- merge_responses: bool # PostJsonSearch for aws_eos
332
- collection: bool # PostJsonSearch for aws_eos
333
- max_connections: int # StaticStacSearch
334
- timeout: float # StaticStacSearch
335
- s3_bucket: str # CreodiasS3Search
336
- end_date_excluded: bool # BuildSearchResult
337
- remove_from_query: List[str] # BuildSearchResult
338
- ssl_verify: bool
339
-
340
- # download -------------------------------------------------------------------------
445
+ #: :class:`~eodag.plugins.search.base.Search` Parameters to be passed as is in the search url query string
446
+ literal_search_params: Dict[str, str]
447
+ #: :class:`~eodag.plugins.search.qssearch.QueryStringSearch` Characters that should not be quoted in the url params
448
+ dont_quote: List[str]
449
+ #: :class:`~eodag.plugins.search.qssearch.ODataV4Search` Dict describing free text search request build
450
+ free_text_search_operations: Dict[str, Any]
451
+ #: :class:`~eodag.plugins.search.qssearch.ODataV4Search` Set to ``True`` if the metadata is not given in the search
452
+ #: result and a two step search has to be performed
453
+ per_product_metadata_query: bool
454
+ #: :class:`~eodag.plugins.search.qssearch.ODataV4Search` Dict used to simplify further metadata extraction
455
+ metadata_pre_mapping: PluginConfig.MetadataPreMapping
456
+ #: :class:`~eodag.plugins.search.data_request_search.DataRequestSearch` URL to which the data request shall be sent
457
+ data_request_url: str
458
+ #: :class:`~eodag.plugins.search.data_request_search.DataRequestSearch` URL to fetch the status of the data request
459
+ status_url: str
460
+ #: :class:`~eodag.plugins.search.data_request_search.DataRequestSearch`
461
+ #: URL to fetch the search result when the data request is done
462
+ result_url: str
463
+ #: :class:`~eodag.plugins.search.data_request_search.DataRequestSearch`
464
+ #: if date parameters are mandatory in the request
465
+ dates_required: bool
466
+ #: :class:`~eodag.plugins.search.csw.CSWSearch` Search definition dictionary
467
+ search_definition: Dict[str, Any]
468
+ #: :class:`~eodag.plugins.search.qssearch.PostJsonSearch` Whether to merge responses or not (`aws_eos` specific)
469
+ merge_responses: bool
470
+ #: :class:`~eodag.plugins.search.qssearch.PostJsonSearch` Collections names (`aws_eos` specific)
471
+ collection: List[str]
472
+ #: :class:`~eodag.plugins.search.static_stac_search.StaticStacSearch`
473
+ #: Maximum number of connections for concurrent HTTP requests
474
+ max_connections: int
475
+ #: :class:`~eodag.plugins.search.build_search_result.ECMWFSearch`
476
+ #: Whether end date should be excluded from search request or not
477
+ end_date_excluded: bool
478
+ #: :class:`~eodag.plugins.search.build_search_result.ECMWFSearch`
479
+ #: List of parameters used to parse metadata but that must not be included to the query
480
+ remove_from_query: List[str]
481
+ #: :class:`~eodag.plugins.search.csw.CSWSearch`
482
+ #: OGC Catalogue Service version
483
+ version: str
484
+ #: :class:`~eodag.plugins.apis.ecmwf.EcmwfApi` url of the authentication endpoint
485
+ auth_endpoint: str
486
+
487
+ # download ---------------------------------------------------------------------------------------------------------
488
+ #: :class:`~eodag.plugins.download.base.Download` Default endpoint url
341
489
  base_uri: str
490
+ #: :class:`~eodag.plugins.download.base.Download` Where to store downloaded products, as an absolute file path
342
491
  output_dir: str
492
+ #: :class:`~eodag.plugins.download.base.Download`
493
+ #: Whether the content of the downloaded file should be extracted or not
343
494
  extract: bool
495
+ #: :class:`~eodag.plugins.download.base.Download` Which extension should be used for the downloaded file
344
496
  output_extension: str
345
- order_enabled: bool # HTTPDownload
346
- order_method: str # HTTPDownload
347
- order_headers: Dict[str, str] # HTTPDownload
348
-
497
+ #: :class:`~eodag.plugins.download.base.Download` Whether the directory structure should be flattened or not
498
+ flatten_top_dirs: bool
499
+ #: :class:`~eodag.plugins.download.base.Download` Level in extracted path tree where to find data
500
+ archive_depth: int
501
+ #: :class:`~eodag.plugins.download.base.Download` Whether ignore assets and download using ``downloadLink`` or not
502
+ ignore_assets: bool
503
+ #: :class:`~eodag.plugins.download.base.Download` Product type specific configuration
504
+ products: Dict[str, Dict[str, Any]]
505
+ #: :class:`~eodag.plugins.download.http.HTTPDownload` Whether the product has to be ordered to download it or not
506
+ order_enabled: bool
507
+ #: :class:`~eodag.plugins.download.http.HTTPDownload` HTTP request method for the order request
508
+ order_method: str
509
+ #: :class:`~eodag.plugins.download.http.HTTPDownload` Headers to be added to the order request
510
+ order_headers: Dict[str, str]
511
+ #: :class:`~eodag.plugins.download.http.HTTPDownload`
512
+ #: Dictionary containing the key :attr:`~eodag.config.PluginConfig.metadata_mapping` which can be used to add new
513
+ #: product properties based on the data in response to the order request
349
514
  order_on_response: PluginConfig.OrderOnResponse
515
+ #: :class:`~eodag.plugins.download.http.HTTPDownload` Order status handling
350
516
  order_status: PluginConfig.OrderStatus
351
- no_auth_download: Annotated[
352
- bool,
353
- Doc(
354
- "Do not authenticate the download request but only the order and order status ones."
355
- ),
356
- ]
357
- bucket_path_level: int # S3RestDownload
358
- requester_pays: bool # AwsDownload
359
- flatten_top_dirs: bool
360
-
361
- # auth -----------------------------------------------------------------------------
517
+ #: :class:`~eodag.plugins.download.http.HTTPDownload`
518
+ #: Do not authenticate the download request but only the order and order status ones
519
+ no_auth_download: bool
520
+ #: :class:`~eodag.plugins.download.http.HTTPDownload` Parameters to be added to the query params of the request
521
+ dl_url_params: Dict[str, str]
522
+ #: :class:`~eodag.plugins.download.s3rest.S3RestDownload`
523
+ #: At which level of the path part of the url the bucket can be found
524
+ bucket_path_level: int
525
+ #: :class:`~eodag.plugins.download.aws.AwsDownload` Whether download is done from a requester-pays bucket or not
526
+ requester_pays: bool
527
+ #: :class:`~eodag.plugins.download.aws.AwsDownload` S3 endpoint
528
+ s3_endpoint: str
529
+
530
+ # auth -------------------------------------------------------------------------------------------------------------
531
+ #: :class:`~eodag.plugins.authentication.base.Authentication` Authentication credentials dictionary
362
532
  credentials: Dict[str, str]
533
+ #: :class:`~eodag.plugins.authentication.base.Authentication` Authentication URL
363
534
  auth_uri: str
364
- auth_base_uri: str
365
- auth_error_code: int
535
+ #: :class:`~eodag.plugins.authentication.base.Authentication`
536
+ #: Dictionary containing all keys/value pairs that should be added to the headers
366
537
  headers: Dict[str, str]
367
- token_provision: str # KeycloakOIDCPasswordAuth
368
- client_id: str # KeycloakOIDCPasswordAuth
369
- client_secret: str # KeycloakOIDCPasswordAuth
370
- realm: str # KeycloakOIDCPasswordAuth
371
- user_consent_needed: str # OIDCAuthorizationCodeFlowAuth
372
- authentication_uri_source: str # OIDCAuthorizationCodeFlowAuth
373
- redirect_uri: str # OIDCAuthorizationCodeFlowAuth
374
- authorization_uri: str # OIDCAuthorizationCodeFlowAuth
375
- login_form_xpath: str # OIDCAuthorizationCodeFlowAuth
376
- user_consent_form_xpath: str # OIDCAuthorizationCodeFlowAuth
377
- user_consent_form_data: Dict[str, str] # OIDCAuthorizationCodeFlowAuth
378
- token_exchange_post_data_method: str # OIDCAuthorizationCodeFlowAuth
379
- token_uri: str # OIDCAuthorizationCodeFlowAuth
380
- token_key: str # OIDCAuthorizationCodeFlowAuth
381
- req_data: Dict[str, Any] # TokenAuth
382
- signed_url_key: str # SASAuth
383
- refresh_uri: str # TokenAuth
384
- refresh_token_key: str # TokenAuth
385
- subject: Dict[str, Any] # TokenExchangeAuth
386
- subject_issuer: str # TokenExchangeAuth
387
- audience: str # TokenExchangeAuth
538
+ #: :class:`~eodag.plugins.authentication.base.Authentication`
539
+ #: Dictionary containing all keys/value pairs that should be added to the headers for token retrieve only
540
+ retrieve_headers: Dict[str, str]
541
+ #: :class:`~eodag.plugins.authentication.base.Authentication`
542
+ #: The key pointing to the token in the response from the token server
543
+ token_key: str
544
+ #: :class:`~eodag.plugins.authentication.base.Authentication`
545
+ #: Key to get the refresh token in the response from the token server
546
+ refresh_token_key: str
547
+ #: :class:`~eodag.plugins.authentication.base.Authentication` URL pattern to match with search plugin endpoint or
548
+ #: download link
549
+ matching_url: str
550
+ #: :class:`~eodag.plugins.authentication.base.Authentication` Part of the search or download plugin configuration
551
+ #: that needs authentication
552
+ matching_conf: Dict[str, Any]
553
+ #: :class:`~eodag.plugins.authentication.openid_connect.OIDCRefreshTokenBase`
554
+ #: How the token should be used in the request
555
+ token_provision: str
556
+ #: :class:`~eodag.plugins.authentication.openid_connect.OIDCRefreshTokenBase` The OIDC provider's client ID
557
+ client_id: str
558
+ #: :class:`~eodag.plugins.authentication.openid_connect.OIDCRefreshTokenBase` The OIDC provider's client secret
559
+ client_secret: str
560
+ #: :class:`~eodag.plugins.authentication.openid_connect.OIDCRefreshTokenBase`
561
+ #: The OIDC provider's ``.well-known/openid-configuration`` url.
562
+ oidc_config_url: str
563
+ #: :class:`~eodag.plugins.authentication.openid_connect.OIDCRefreshTokenBase` The OIDC token audiences
564
+ allowed_audiences: List[str]
565
+ #: :class:`~eodag.plugins.authentication.openid_connect.OIDCAuthorizationCodeFlowAuth`
566
+ #: Whether a user consent is needed during the authentication or not
567
+ user_consent_needed: str
568
+ #: :class:`~eodag.plugins.authentication.openid_connect.OIDCAuthorizationCodeFlowAuth`
569
+ #: Where to look for the :attr:`~eodag.config.PluginConfig.authorization_uri`
570
+ authentication_uri_source: str
571
+ #: :class:`~eodag.plugins.authentication.openid_connect.OIDCAuthorizationCodeFlowAuth`
572
+ #: The callback url that will handle the code given by the OIDC provider
573
+ authentication_uri: str
574
+ #: :class:`~eodag.plugins.authentication.openid_connect.OIDCAuthorizationCodeFlowAuth`
575
+ #: The URL of the authentication backend of the OIDC provider
576
+ redirect_uri: str
577
+ #: :class:`~eodag.plugins.authentication.openid_connect.OIDCAuthorizationCodeFlowAuth`
578
+ #: The authorization url of the server (where to query for grants)
579
+ authorization_uri: str
580
+ #: :class:`~eodag.plugins.authentication.openid_connect.OIDCAuthorizationCodeFlowAuth`
581
+ #: The xpath to the HTML form element representing the user login form
582
+ login_form_xpath: str
583
+ #: :class:`~eodag.plugins.authentication.openid_connect.OIDCAuthorizationCodeFlowAuth`
584
+ #: The xpath to the user consent form
585
+ user_consent_form_xpath: str
586
+ #: :class:`~eodag.plugins.authentication.openid_connect.OIDCAuthorizationCodeFlowAuth`
587
+ #: The data that will be passed with the POST request on the form 'action' URL
588
+ user_consent_form_data: Dict[str, str]
589
+ #: :class:`~eodag.plugins.authentication.openid_connect.OIDCAuthorizationCodeFlowAuth`
590
+ #: Additional data to be passed to the login POST request
591
+ additional_login_form_data: Dict[str, str]
592
+ #: :class:`~eodag.plugins.authentication.openid_connect.OIDCAuthorizationCodeFlowAuth`
593
+ #: Key/value pairs of patterns/messages used for Authentication errors
594
+ exchange_url_error_pattern: Dict[str, str]
595
+ #: :class:`~eodag.plugins.authentication.openid_connect.OIDCAuthorizationCodeFlowAuth`
596
+ #: A mapping between OIDC url query string and token handler query string params
597
+ token_exchange_params: Dict[str, str]
598
+ #: :class:`~eodag.plugins.authentication.openid_connect.OIDCAuthorizationCodeFlowAuth`
599
+ #: Refers to the name of the query param to be used in the query request
600
+ token_qs_key: str
601
+ #: :class:`~eodag.plugins.authentication.openid_connect.OIDCAuthorizationCodeFlowAuth`
602
+ #: Way to pass the data to the POST request that is made to the token server
603
+ token_exchange_post_data_method: str
604
+ #: :class:`~eodag.plugins.authentication.openid_connect.OIDCAuthorizationCodeFlowAuth`
605
+ #: The url to query to get the authorized token
606
+ token_uri: str
607
+ #: :class:`~eodag.plugins.authentication.sas_auth.SASAuth` Key to get the signed url
608
+ signed_url_key: str
609
+ #: :class:`~eodag.plugins.authentication.token.TokenAuth`
610
+ #: Credentials json structure if they should be sent as POST data
611
+ req_data: Dict[str, Any]
612
+ #: :class:`~eodag.plugins.authentication.token.TokenAuth`
613
+ #: URL used to fetch the access token with a refresh token
614
+ refresh_uri: str
615
+ #: :class:`~eodag.plugins.authentication.token.TokenAuth`
616
+ #: type of the token
617
+ token_type: str
618
+ #: :class:`~eodag.plugins.authentication.token_exchange.OIDCTokenExchangeAuth`
619
+ #: The full :class:`~eodag.plugins.authentication.openid_connect.OIDCAuthorizationCodeFlowAuth` plugin configuration
620
+ #: used to retrieve subject token
621
+ subject: Dict[str, Any]
622
+ #: :class:`~eodag.plugins.authentication.token_exchange.OIDCTokenExchangeAuth`
623
+ #: Identifies the issuer of the `subject_token`
624
+ subject_issuer: str
625
+ #: :class:`~eodag.plugins.authentication.token_exchange.OIDCTokenExchangeAuth`
626
+ #: Audience that the token ID is intended for. :attr:`~eodag.config.PluginConfig.client_id` of the Relying Party
627
+ audience: str
628
+ #: :class:`~eodag.plugins.authentication.generic.GenericAuth`
629
+ #: which authentication method should be used
630
+ method: str
388
631
 
389
632
  yaml_loader = yaml.Loader
390
633
  yaml_dumper = yaml.SafeDumper
@@ -424,7 +667,7 @@ class PluginConfig(yaml.YAMLObject):
424
667
 
425
668
 
426
669
  def load_default_config() -> Dict[str, ProviderConfig]:
427
- """Load the providers configuration into a dictionnary.
670
+ """Load the providers configuration into a dictionary.
428
671
 
429
672
  Load from eodag `resources/providers.yml` or `EODAG_PROVIDERS_CFG_FILE` environment
430
673
  variable if exists.
@@ -438,7 +681,7 @@ def load_default_config() -> Dict[str, ProviderConfig]:
438
681
 
439
682
 
440
683
  def load_config(config_path: str) -> Dict[str, ProviderConfig]:
441
- """Load the providers configuration into a dictionnary from a given file
684
+ """Load the providers configuration into a dictionary from a given file
442
685
 
443
686
  :param config_path: The path to the provider config file
444
687
  :returns: The default provider's configuration
@@ -454,12 +697,65 @@ def load_config(config_path: str) -> Dict[str, ProviderConfig]:
454
697
  raise e
455
698
  stac_provider_config = load_stac_provider_config()
456
699
  for provider_config in providers_configs:
457
- # for provider_config in copy.deepcopy(providers_configs):
458
700
  provider_config_init(provider_config, stac_provider_config)
459
701
  config[provider_config.name] = provider_config
460
702
  return config
461
703
 
462
704
 
705
+ def credentials_in_auth(auth_conf: PluginConfig) -> bool:
706
+ """Checks if credentials are set for this Authentication plugin configuration
707
+
708
+ :param auth_conf: Authentication plugin configuration
709
+ :returns: True if credentials are set, else False
710
+ """
711
+ return any(
712
+ c is not None for c in (getattr(auth_conf, "credentials", {}) or {}).values()
713
+ )
714
+
715
+
716
+ def share_credentials(
717
+ providers_config: Dict[str, ProviderConfig],
718
+ ) -> None:
719
+ """Share credentials between plugins having the same matching criteria
720
+
721
+ :param providers_configs: eodag providers configurations
722
+ """
723
+ auth_confs_with_creds = [
724
+ getattr(p, k)
725
+ for p in providers_config.values()
726
+ for k in AUTH_TOPIC_KEYS
727
+ if hasattr(p, k) and credentials_in_auth(getattr(p, k))
728
+ ]
729
+ for provider, provider_config in providers_config.items():
730
+ if auth_confs_with_creds:
731
+ for auth_topic_key in AUTH_TOPIC_KEYS:
732
+ provider_config_auth = getattr(provider_config, auth_topic_key, None)
733
+ if provider_config_auth and not credentials_in_auth(
734
+ provider_config_auth
735
+ ):
736
+ # no credentials set for this provider
737
+ provider_matching_conf = getattr(
738
+ provider_config_auth, "matching_conf", {}
739
+ )
740
+ provider_matching_url = getattr(
741
+ provider_config_auth, "matching_url", None
742
+ )
743
+ for conf_with_creds in auth_confs_with_creds:
744
+ # copy credentials between plugins if `matching_conf` or `matching_url` are matching
745
+ if (
746
+ provider_matching_conf
747
+ and sort_dict(provider_matching_conf)
748
+ == sort_dict(getattr(conf_with_creds, "matching_conf", {}))
749
+ ) or (
750
+ provider_matching_url
751
+ and provider_matching_url
752
+ == getattr(conf_with_creds, "matching_url", None)
753
+ ):
754
+ getattr(
755
+ providers_config[provider], auth_topic_key
756
+ ).credentials = conf_with_creds.credentials
757
+
758
+
463
759
  def provider_config_init(
464
760
  provider_config: ProviderConfig,
465
761
  stac_search_default_conf: Optional[Dict[str, Any]] = None,
@@ -471,13 +767,13 @@ def provider_config_init(
471
767
  """
472
768
  # For the provider, set the default output_dir of its download plugin
473
769
  # as tempdir in a portable way
474
- for param_name in ("download", "api"):
475
- if param_name in vars(provider_config):
476
- param_value = getattr(provider_config, param_name)
477
- if not getattr(param_value, "output_dir", None):
478
- param_value.output_dir = tempfile.gettempdir()
479
- if not getattr(param_value, "delete_archive", None):
480
- param_value.delete_archive = True
770
+ for download_topic_key in ("download", "api"):
771
+ if download_topic_key in vars(provider_config):
772
+ download_conf = getattr(provider_config, download_topic_key)
773
+ if not getattr(download_conf, "output_dir", None):
774
+ download_conf.output_dir = tempfile.gettempdir()
775
+ if not getattr(download_conf, "delete_archive", None):
776
+ download_conf.delete_archive = True
481
777
 
482
778
  try:
483
779
  if (
@@ -681,7 +977,7 @@ def merge_configs(config: Dict[str, Any], other_config: Dict[str, Any]) -> None:
681
977
 
682
978
 
683
979
  def load_yml_config(yml_path: str) -> Dict[Any, Any]:
684
- """Load a conf dictionnary from given yml absolute path
980
+ """Load a conf dictionary from given yml absolute path
685
981
 
686
982
  :returns: The yml configuration file
687
983
  """
@@ -690,7 +986,7 @@ def load_yml_config(yml_path: str) -> Dict[Any, Any]:
690
986
 
691
987
 
692
988
  def load_stac_config() -> Dict[str, Any]:
693
- """Load the stac configuration into a dictionnary
989
+ """Load the stac configuration into a dictionary
694
990
 
695
991
  :returns: The stac configuration
696
992
  """
@@ -700,7 +996,7 @@ def load_stac_config() -> Dict[str, Any]:
700
996
 
701
997
 
702
998
  def load_stac_api_config() -> Dict[str, Any]:
703
- """Load the stac API configuration into a dictionnary
999
+ """Load the stac API configuration into a dictionary
704
1000
 
705
1001
  :returns: The stac API configuration
706
1002
  """
@@ -710,7 +1006,7 @@ def load_stac_api_config() -> Dict[str, Any]:
710
1006
 
711
1007
 
712
1008
  def load_stac_provider_config() -> Dict[str, Any]:
713
- """Load the stac provider configuration into a dictionnary
1009
+ """Load the stac provider configuration into a dictionary
714
1010
 
715
1011
  :returns: The stac provider configuration
716
1012
  """