eodag 3.1.0b1__py3-none-any.whl → 3.1.0b2__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- eodag/api/core.py +59 -52
- eodag/api/product/_assets.py +5 -5
- eodag/api/product/_product.py +27 -12
- eodag/api/product/drivers/__init__.py +81 -4
- eodag/api/product/drivers/base.py +65 -4
- eodag/api/product/drivers/generic.py +65 -0
- eodag/api/product/drivers/sentinel1.py +97 -0
- eodag/api/product/drivers/sentinel2.py +95 -0
- eodag/api/product/metadata_mapping.py +62 -74
- eodag/api/search_result.py +13 -23
- eodag/cli.py +4 -4
- eodag/config.py +66 -69
- eodag/plugins/apis/base.py +1 -1
- eodag/plugins/apis/ecmwf.py +10 -9
- eodag/plugins/apis/usgs.py +11 -10
- eodag/plugins/authentication/aws_auth.py +16 -13
- eodag/plugins/authentication/base.py +5 -3
- eodag/plugins/authentication/header.py +3 -3
- eodag/plugins/authentication/keycloak.py +4 -4
- eodag/plugins/authentication/oauth.py +7 -3
- eodag/plugins/authentication/openid_connect.py +14 -14
- eodag/plugins/authentication/sas_auth.py +4 -4
- eodag/plugins/authentication/token.py +7 -7
- eodag/plugins/authentication/token_exchange.py +1 -1
- eodag/plugins/base.py +4 -4
- eodag/plugins/crunch/base.py +4 -4
- eodag/plugins/crunch/filter_date.py +4 -4
- eodag/plugins/crunch/filter_latest_intersect.py +6 -6
- eodag/plugins/crunch/filter_latest_tpl_name.py +7 -7
- eodag/plugins/crunch/filter_overlap.py +4 -4
- eodag/plugins/crunch/filter_property.py +4 -4
- eodag/plugins/download/aws.py +47 -66
- eodag/plugins/download/base.py +8 -17
- eodag/plugins/download/creodias_s3.py +2 -2
- eodag/plugins/download/http.py +30 -32
- eodag/plugins/download/s3rest.py +5 -4
- eodag/plugins/manager.py +10 -20
- eodag/plugins/search/__init__.py +6 -5
- eodag/plugins/search/base.py +35 -40
- eodag/plugins/search/build_search_result.py +69 -68
- eodag/plugins/search/cop_marine.py +22 -12
- eodag/plugins/search/creodias_s3.py +8 -78
- eodag/plugins/search/csw.py +11 -11
- eodag/plugins/search/data_request_search.py +16 -15
- eodag/plugins/search/qssearch.py +56 -52
- eodag/plugins/search/stac_list_assets.py +85 -0
- eodag/plugins/search/static_stac_search.py +3 -3
- eodag/resources/ext_product_types.json +1 -1
- eodag/resources/product_types.yml +288 -288
- eodag/resources/providers.yml +146 -6
- eodag/resources/stac_api.yml +2 -2
- eodag/resources/user_conf_template.yml +11 -0
- eodag/rest/cache.py +2 -2
- eodag/rest/config.py +3 -3
- eodag/rest/core.py +24 -24
- eodag/rest/errors.py +5 -5
- eodag/rest/server.py +3 -11
- eodag/rest/stac.py +40 -38
- eodag/rest/types/collections_search.py +3 -3
- eodag/rest/types/eodag_search.py +23 -23
- eodag/rest/types/queryables.py +13 -13
- eodag/rest/types/stac_search.py +15 -25
- eodag/rest/utils/__init__.py +11 -21
- eodag/rest/utils/cql_evaluate.py +6 -6
- eodag/rest/utils/rfc3339.py +2 -2
- eodag/types/__init__.py +24 -18
- eodag/types/bbox.py +2 -2
- eodag/types/download_args.py +2 -2
- eodag/types/queryables.py +5 -2
- eodag/types/search_args.py +4 -4
- eodag/types/whoosh.py +1 -3
- eodag/utils/__init__.py +81 -40
- eodag/utils/exceptions.py +2 -2
- eodag/utils/import_system.py +2 -2
- eodag/utils/requests.py +2 -2
- eodag/utils/rest.py +2 -2
- eodag/utils/s3.py +208 -0
- eodag/utils/stac_reader.py +10 -10
- {eodag-3.1.0b1.dist-info → eodag-3.1.0b2.dist-info}/METADATA +5 -4
- eodag-3.1.0b2.dist-info/RECORD +113 -0
- {eodag-3.1.0b1.dist-info → eodag-3.1.0b2.dist-info}/entry_points.txt +1 -0
- eodag-3.1.0b1.dist-info/RECORD +0 -108
- {eodag-3.1.0b1.dist-info → eodag-3.1.0b2.dist-info}/LICENSE +0 -0
- {eodag-3.1.0b1.dist-info → eodag-3.1.0b2.dist-info}/WHEEL +0 -0
- {eodag-3.1.0b1.dist-info → eodag-3.1.0b2.dist-info}/top_level.txt +0 -0
eodag/config.py
CHANGED
|
@@ -24,13 +24,10 @@ from inspect import isclass
|
|
|
24
24
|
from typing import (
|
|
25
25
|
Annotated,
|
|
26
26
|
Any,
|
|
27
|
-
Dict,
|
|
28
27
|
ItemsView,
|
|
29
28
|
Iterator,
|
|
30
|
-
List,
|
|
31
29
|
Literal,
|
|
32
30
|
Optional,
|
|
33
|
-
Tuple,
|
|
34
31
|
TypedDict,
|
|
35
32
|
Union,
|
|
36
33
|
ValuesView,
|
|
@@ -79,7 +76,7 @@ class SimpleYamlProxyConfig:
|
|
|
79
76
|
|
|
80
77
|
def __init__(self, conf_file_path: str) -> None:
|
|
81
78
|
try:
|
|
82
|
-
self.source:
|
|
79
|
+
self.source: dict[str, Any] = cached_yaml_load(conf_file_path)
|
|
83
80
|
except yaml.parser.ParserError as e:
|
|
84
81
|
print("Unable to load user configuration file")
|
|
85
82
|
raise e
|
|
@@ -127,12 +124,12 @@ class ProviderConfig(yaml.YAMLObject):
|
|
|
127
124
|
name: str
|
|
128
125
|
group: str
|
|
129
126
|
priority: int = 0 # Set default priority to 0
|
|
130
|
-
roles:
|
|
127
|
+
roles: list[str]
|
|
131
128
|
description: str
|
|
132
129
|
url: str
|
|
133
130
|
api: PluginConfig
|
|
134
131
|
search: PluginConfig
|
|
135
|
-
products:
|
|
132
|
+
products: dict[str, Any]
|
|
136
133
|
download: PluginConfig
|
|
137
134
|
auth: PluginConfig
|
|
138
135
|
search_auth: PluginConfig
|
|
@@ -154,7 +151,7 @@ class ProviderConfig(yaml.YAMLObject):
|
|
|
154
151
|
return loader.construct_yaml_object(node, cls)
|
|
155
152
|
|
|
156
153
|
@classmethod
|
|
157
|
-
def from_mapping(cls, mapping:
|
|
154
|
+
def from_mapping(cls, mapping: dict[str, Any]) -> ProviderConfig:
|
|
158
155
|
"""Build a :class:`~eodag.config.ProviderConfig` from a mapping"""
|
|
159
156
|
cls.validate(mapping)
|
|
160
157
|
for key in PLUGINS_TOPICS_KEYS:
|
|
@@ -165,7 +162,7 @@ class ProviderConfig(yaml.YAMLObject):
|
|
|
165
162
|
return c
|
|
166
163
|
|
|
167
164
|
@staticmethod
|
|
168
|
-
def validate(config_keys: Union[
|
|
165
|
+
def validate(config_keys: Union[tuple[str, ...], dict[str, Any]]) -> None:
|
|
169
166
|
"""Validate a :class:`~eodag.config.ProviderConfig`
|
|
170
167
|
|
|
171
168
|
:param config_keys: The configurations keys to validate
|
|
@@ -181,7 +178,7 @@ class ProviderConfig(yaml.YAMLObject):
|
|
|
181
178
|
"type of plugin"
|
|
182
179
|
)
|
|
183
180
|
|
|
184
|
-
def update(self, mapping: Optional[
|
|
181
|
+
def update(self, mapping: Optional[dict[str, Any]]) -> None:
|
|
185
182
|
"""Update the configuration parameters with values from `mapping`
|
|
186
183
|
|
|
187
184
|
:param mapping: The mapping from which to override configuration parameters
|
|
@@ -197,7 +194,7 @@ class ProviderConfig(yaml.YAMLObject):
|
|
|
197
194
|
},
|
|
198
195
|
)
|
|
199
196
|
for key in PLUGINS_TOPICS_KEYS:
|
|
200
|
-
current_value: Optional[
|
|
197
|
+
current_value: Optional[dict[str, Any]] = getattr(self, key, None)
|
|
201
198
|
mapping_value = mapping.get(key, {})
|
|
202
199
|
if current_value is not None:
|
|
203
200
|
current_value.update(mapping_value)
|
|
@@ -240,14 +237,14 @@ class PluginConfig(yaml.YAMLObject):
|
|
|
240
237
|
"""Configuration for sort during search"""
|
|
241
238
|
|
|
242
239
|
#: Default sort settings
|
|
243
|
-
sort_by_default:
|
|
240
|
+
sort_by_default: list[tuple[str, str]]
|
|
244
241
|
#: F-string template to add to :attr:`~eodag.config.PluginConfig.Pagination.next_page_url_tpl` to sort search
|
|
245
242
|
#: results
|
|
246
243
|
sort_by_tpl: str
|
|
247
244
|
#: Mapping between eodag and provider query parameters used for sort
|
|
248
|
-
sort_param_mapping:
|
|
245
|
+
sort_param_mapping: dict[str, str]
|
|
249
246
|
#: Mapping between eodag and provider sort-order parameters
|
|
250
|
-
sort_order_mapping:
|
|
247
|
+
sort_order_mapping: dict[Literal["ascending", "descending"], str]
|
|
251
248
|
#: Maximum number of allowed sort parameters per request
|
|
252
249
|
max_sort_params: Annotated[int, Gt(0)]
|
|
253
250
|
|
|
@@ -271,7 +268,7 @@ class PluginConfig(yaml.YAMLObject):
|
|
|
271
268
|
#: HTTP method used to fetch product types
|
|
272
269
|
fetch_method: str
|
|
273
270
|
#: Request body to fetch product types using POST method
|
|
274
|
-
fetch_body:
|
|
271
|
+
fetch_body: dict[str, Any]
|
|
275
272
|
#: Maximum number of connections for concurrent HTTP requests
|
|
276
273
|
max_connections: int
|
|
277
274
|
#: The f-string template for pagination requests.
|
|
@@ -286,17 +283,17 @@ class PluginConfig(yaml.YAMLObject):
|
|
|
286
283
|
generic_product_type_id: str
|
|
287
284
|
#: Mapping for product type metadata (e.g. ``abstract``, ``licence``) which can be parsed from the provider
|
|
288
285
|
#: result
|
|
289
|
-
generic_product_type_parsable_metadata:
|
|
286
|
+
generic_product_type_parsable_metadata: dict[str, str]
|
|
290
287
|
#: Mapping for product type properties which can be parsed from the result and are not product type metadata
|
|
291
|
-
generic_product_type_parsable_properties:
|
|
288
|
+
generic_product_type_parsable_properties: dict[str, str]
|
|
292
289
|
#: Mapping for product type properties which cannot be parsed from the result and are not product type metadata
|
|
293
|
-
generic_product_type_unparsable_properties:
|
|
290
|
+
generic_product_type_unparsable_properties: dict[str, str]
|
|
294
291
|
#: URL to fetch data for a single collection
|
|
295
292
|
single_collection_fetch_url: str
|
|
296
293
|
#: Query string to be added to the fetch_url to filter for a collection
|
|
297
294
|
single_collection_fetch_qs: str
|
|
298
295
|
#: Mapping for product type metadata returned by the endpoint given in single_collection_fetch_url
|
|
299
|
-
single_product_type_parsable_metadata:
|
|
296
|
+
single_product_type_parsable_metadata: dict[str, str]
|
|
300
297
|
|
|
301
298
|
class DiscoverQueryables(TypedDict, total=False):
|
|
302
299
|
"""Configuration for queryables discovery"""
|
|
@@ -318,7 +315,7 @@ class PluginConfig(yaml.YAMLObject):
|
|
|
318
315
|
"""Configuration for order on-response during download"""
|
|
319
316
|
|
|
320
317
|
#: Parameters metadata-mapping to apply to the order response
|
|
321
|
-
metadata_mapping:
|
|
318
|
+
metadata_mapping: dict[str, Union[str, list[str]]]
|
|
322
319
|
|
|
323
320
|
class OrderStatusSuccess(TypedDict):
|
|
324
321
|
"""
|
|
@@ -351,7 +348,7 @@ class PluginConfig(yaml.YAMLObject):
|
|
|
351
348
|
#: Request HTTP method
|
|
352
349
|
method: str
|
|
353
350
|
#: Request hearders
|
|
354
|
-
headers:
|
|
351
|
+
headers: dict[str, Any]
|
|
355
352
|
|
|
356
353
|
class OrderStatusOnSuccess(TypedDict, total=False):
|
|
357
354
|
"""Configuration for order status on-success during download"""
|
|
@@ -363,7 +360,7 @@ class PluginConfig(yaml.YAMLObject):
|
|
|
363
360
|
#: Key in the success response that gives access to the result
|
|
364
361
|
results_entry: str
|
|
365
362
|
#: Metadata-mapping to apply to the success status result
|
|
366
|
-
metadata_mapping:
|
|
363
|
+
metadata_mapping: dict[str, Union[str, list[str]]]
|
|
367
364
|
|
|
368
365
|
class OrderStatus(TypedDict, total=False):
|
|
369
366
|
"""Configuration for order status during download"""
|
|
@@ -371,11 +368,11 @@ class PluginConfig(yaml.YAMLObject):
|
|
|
371
368
|
#: Order status request configuration
|
|
372
369
|
request: PluginConfig.OrderStatusRequest
|
|
373
370
|
#: Metadata-mapping used to parse order status response
|
|
374
|
-
metadata_mapping:
|
|
371
|
+
metadata_mapping: dict[str, Union[str, list[str]]]
|
|
375
372
|
#: Configuration to identify order status success during download
|
|
376
373
|
success: PluginConfig.OrderStatusSuccess
|
|
377
374
|
#: Part of the order status response that tells there is an error
|
|
378
|
-
error:
|
|
375
|
+
error: dict[str, Any]
|
|
379
376
|
#: Configuration to identify order status ordered during download
|
|
380
377
|
ordered: PluginConfig.OrderStatusOrdered
|
|
381
378
|
#: Configuration for order status on-success during download
|
|
@@ -400,7 +397,7 @@ class PluginConfig(yaml.YAMLObject):
|
|
|
400
397
|
#: :class:`~eodag.plugins.base.PluginTopic` Default s3 bucket
|
|
401
398
|
s3_bucket: str
|
|
402
399
|
#: :class:`~eodag.plugins.base.PluginTopic` Authentication error codes
|
|
403
|
-
auth_error_code: Union[int,
|
|
400
|
+
auth_error_code: Union[int, list[int]]
|
|
404
401
|
#: :class:`~eodag.plugins.base.PluginTopic` Time to wait until request timeout in seconds
|
|
405
402
|
timeout: float
|
|
406
403
|
#: :class:`~eodag.plugins.base.PluginTopic` :class:`urllib3.util.Retry` ``total`` parameter,
|
|
@@ -411,13 +408,13 @@ class PluginConfig(yaml.YAMLObject):
|
|
|
411
408
|
retry_backoff_factor: int
|
|
412
409
|
#: :class:`~eodag.plugins.base.PluginTopic` :class:`urllib3.util.Retry` ``status_forcelist`` parameter,
|
|
413
410
|
#: list of integer HTTP status codes that we should force a retry on
|
|
414
|
-
retry_status_forcelist:
|
|
411
|
+
retry_status_forcelist: list[int]
|
|
415
412
|
|
|
416
413
|
# search & api -----------------------------------------------------------------------------------------------------
|
|
417
414
|
# copied from ProviderConfig in PluginManager.get_search_plugins()
|
|
418
415
|
priority: int
|
|
419
416
|
# per product type metadata-mapping, set in core._prepare_search
|
|
420
|
-
product_type_config:
|
|
417
|
+
product_type_config: dict[str, Any]
|
|
421
418
|
|
|
422
419
|
#: :class:`~eodag.plugins.search.base.Search` Plugin API endpoint
|
|
423
420
|
api_endpoint: str
|
|
@@ -439,15 +436,15 @@ class PluginConfig(yaml.YAMLObject):
|
|
|
439
436
|
#: :class:`~eodag.plugins.search.base.Search` Configuration for the queryables auto-discovery
|
|
440
437
|
discover_queryables: PluginConfig.DiscoverQueryables
|
|
441
438
|
#: :class:`~eodag.plugins.search.base.Search` The mapping between eodag metadata and the plugin specific metadata
|
|
442
|
-
metadata_mapping:
|
|
439
|
+
metadata_mapping: dict[str, Union[str, list[str]]]
|
|
443
440
|
#: :class:`~eodag.plugins.search.base.Search` Parameters to remove from queryables
|
|
444
|
-
remove_from_queryables:
|
|
441
|
+
remove_from_queryables: list[str]
|
|
445
442
|
#: :class:`~eodag.plugins.search.base.Search` Parameters to be passed as is in the search url query string
|
|
446
|
-
literal_search_params:
|
|
443
|
+
literal_search_params: dict[str, str]
|
|
447
444
|
#: :class:`~eodag.plugins.search.qssearch.QueryStringSearch` Characters that should not be quoted in the url params
|
|
448
|
-
dont_quote:
|
|
445
|
+
dont_quote: list[str]
|
|
449
446
|
#: :class:`~eodag.plugins.search.qssearch.ODataV4Search` Dict describing free text search request build
|
|
450
|
-
free_text_search_operations:
|
|
447
|
+
free_text_search_operations: dict[str, Any]
|
|
451
448
|
#: :class:`~eodag.plugins.search.qssearch.ODataV4Search` Set to ``True`` if the metadata is not given in the search
|
|
452
449
|
#: result and a two step search has to be performed
|
|
453
450
|
per_product_metadata_query: bool
|
|
@@ -464,11 +461,11 @@ class PluginConfig(yaml.YAMLObject):
|
|
|
464
461
|
#: if date parameters are mandatory in the request
|
|
465
462
|
dates_required: bool
|
|
466
463
|
#: :class:`~eodag.plugins.search.csw.CSWSearch` Search definition dictionary
|
|
467
|
-
search_definition:
|
|
464
|
+
search_definition: dict[str, Any]
|
|
468
465
|
#: :class:`~eodag.plugins.search.qssearch.PostJsonSearch` Whether to merge responses or not (`aws_eos` specific)
|
|
469
466
|
merge_responses: bool
|
|
470
467
|
#: :class:`~eodag.plugins.search.qssearch.PostJsonSearch` Collections names (`aws_eos` specific)
|
|
471
|
-
collection:
|
|
468
|
+
collection: list[str]
|
|
472
469
|
#: :class:`~eodag.plugins.search.static_stac_search.StaticStacSearch`
|
|
473
470
|
#: Maximum number of connections for concurrent HTTP requests
|
|
474
471
|
max_connections: int
|
|
@@ -477,7 +474,7 @@ class PluginConfig(yaml.YAMLObject):
|
|
|
477
474
|
end_date_excluded: bool
|
|
478
475
|
#: :class:`~eodag.plugins.search.build_search_result.ECMWFSearch`
|
|
479
476
|
#: List of parameters used to parse metadata but that must not be included to the query
|
|
480
|
-
remove_from_query:
|
|
477
|
+
remove_from_query: list[str]
|
|
481
478
|
#: :class:`~eodag.plugins.search.csw.CSWSearch`
|
|
482
479
|
#: OGC Catalogue Service version
|
|
483
480
|
version: str
|
|
@@ -501,13 +498,13 @@ class PluginConfig(yaml.YAMLObject):
|
|
|
501
498
|
#: :class:`~eodag.plugins.download.base.Download` Whether ignore assets and download using ``downloadLink`` or not
|
|
502
499
|
ignore_assets: bool
|
|
503
500
|
#: :class:`~eodag.plugins.download.base.Download` Product type specific configuration
|
|
504
|
-
products:
|
|
501
|
+
products: dict[str, dict[str, Any]]
|
|
505
502
|
#: :class:`~eodag.plugins.download.http.HTTPDownload` Whether the product has to be ordered to download it or not
|
|
506
503
|
order_enabled: bool
|
|
507
504
|
#: :class:`~eodag.plugins.download.http.HTTPDownload` HTTP request method for the order request
|
|
508
505
|
order_method: str
|
|
509
506
|
#: :class:`~eodag.plugins.download.http.HTTPDownload` Headers to be added to the order request
|
|
510
|
-
order_headers:
|
|
507
|
+
order_headers: dict[str, str]
|
|
511
508
|
#: :class:`~eodag.plugins.download.http.HTTPDownload`
|
|
512
509
|
#: Dictionary containing the key :attr:`~eodag.config.PluginConfig.metadata_mapping` which can be used to add new
|
|
513
510
|
#: product properties based on the data in response to the order request
|
|
@@ -518,7 +515,7 @@ class PluginConfig(yaml.YAMLObject):
|
|
|
518
515
|
#: Do not authenticate the download request but only the order and order status ones
|
|
519
516
|
no_auth_download: bool
|
|
520
517
|
#: :class:`~eodag.plugins.download.http.HTTPDownload` Parameters to be added to the query params of the request
|
|
521
|
-
dl_url_params:
|
|
518
|
+
dl_url_params: dict[str, str]
|
|
522
519
|
#: :class:`~eodag.plugins.download.s3rest.S3RestDownload`
|
|
523
520
|
#: At which level of the path part of the url the bucket can be found
|
|
524
521
|
bucket_path_level: int
|
|
@@ -529,15 +526,15 @@ class PluginConfig(yaml.YAMLObject):
|
|
|
529
526
|
|
|
530
527
|
# auth -------------------------------------------------------------------------------------------------------------
|
|
531
528
|
#: :class:`~eodag.plugins.authentication.base.Authentication` Authentication credentials dictionary
|
|
532
|
-
credentials:
|
|
529
|
+
credentials: dict[str, str]
|
|
533
530
|
#: :class:`~eodag.plugins.authentication.base.Authentication` Authentication URL
|
|
534
531
|
auth_uri: str
|
|
535
532
|
#: :class:`~eodag.plugins.authentication.base.Authentication`
|
|
536
533
|
#: Dictionary containing all keys/value pairs that should be added to the headers
|
|
537
|
-
headers:
|
|
534
|
+
headers: dict[str, str]
|
|
538
535
|
#: :class:`~eodag.plugins.authentication.base.Authentication`
|
|
539
536
|
#: Dictionary containing all keys/value pairs that should be added to the headers for token retrieve only
|
|
540
|
-
retrieve_headers:
|
|
537
|
+
retrieve_headers: dict[str, str]
|
|
541
538
|
#: :class:`~eodag.plugins.authentication.base.Authentication`
|
|
542
539
|
#: The key pointing to the token in the response from the token server
|
|
543
540
|
token_key: str
|
|
@@ -549,7 +546,7 @@ class PluginConfig(yaml.YAMLObject):
|
|
|
549
546
|
matching_url: str
|
|
550
547
|
#: :class:`~eodag.plugins.authentication.base.Authentication` Part of the search or download plugin configuration
|
|
551
548
|
#: that needs authentication
|
|
552
|
-
matching_conf:
|
|
549
|
+
matching_conf: dict[str, Any]
|
|
553
550
|
#: :class:`~eodag.plugins.authentication.openid_connect.OIDCRefreshTokenBase`
|
|
554
551
|
#: How the token should be used in the request
|
|
555
552
|
token_provision: str
|
|
@@ -561,7 +558,7 @@ class PluginConfig(yaml.YAMLObject):
|
|
|
561
558
|
#: The OIDC provider's ``.well-known/openid-configuration`` url.
|
|
562
559
|
oidc_config_url: str
|
|
563
560
|
#: :class:`~eodag.plugins.authentication.openid_connect.OIDCRefreshTokenBase` The OIDC token audiences
|
|
564
|
-
allowed_audiences:
|
|
561
|
+
allowed_audiences: list[str]
|
|
565
562
|
#: :class:`~eodag.plugins.authentication.openid_connect.OIDCAuthorizationCodeFlowAuth`
|
|
566
563
|
#: Whether a user consent is needed during the authentication or not
|
|
567
564
|
user_consent_needed: str
|
|
@@ -585,16 +582,16 @@ class PluginConfig(yaml.YAMLObject):
|
|
|
585
582
|
user_consent_form_xpath: str
|
|
586
583
|
#: :class:`~eodag.plugins.authentication.openid_connect.OIDCAuthorizationCodeFlowAuth`
|
|
587
584
|
#: The data that will be passed with the POST request on the form 'action' URL
|
|
588
|
-
user_consent_form_data:
|
|
585
|
+
user_consent_form_data: dict[str, str]
|
|
589
586
|
#: :class:`~eodag.plugins.authentication.openid_connect.OIDCAuthorizationCodeFlowAuth`
|
|
590
587
|
#: Additional data to be passed to the login POST request
|
|
591
|
-
additional_login_form_data:
|
|
588
|
+
additional_login_form_data: dict[str, str]
|
|
592
589
|
#: :class:`~eodag.plugins.authentication.openid_connect.OIDCAuthorizationCodeFlowAuth`
|
|
593
590
|
#: Key/value pairs of patterns/messages used for Authentication errors
|
|
594
|
-
exchange_url_error_pattern:
|
|
591
|
+
exchange_url_error_pattern: dict[str, str]
|
|
595
592
|
#: :class:`~eodag.plugins.authentication.openid_connect.OIDCAuthorizationCodeFlowAuth`
|
|
596
593
|
#: A mapping between OIDC url query string and token handler query string params
|
|
597
|
-
token_exchange_params:
|
|
594
|
+
token_exchange_params: dict[str, str]
|
|
598
595
|
#: :class:`~eodag.plugins.authentication.openid_connect.OIDCAuthorizationCodeFlowAuth`
|
|
599
596
|
#: Refers to the name of the query param to be used in the query request
|
|
600
597
|
token_qs_key: str
|
|
@@ -608,7 +605,7 @@ class PluginConfig(yaml.YAMLObject):
|
|
|
608
605
|
signed_url_key: str
|
|
609
606
|
#: :class:`~eodag.plugins.authentication.token.TokenAuth`
|
|
610
607
|
#: Credentials json structure if they should be sent as POST data
|
|
611
|
-
req_data:
|
|
608
|
+
req_data: dict[str, Any]
|
|
612
609
|
#: :class:`~eodag.plugins.authentication.token.TokenAuth`
|
|
613
610
|
#: URL used to fetch the access token with a refresh token
|
|
614
611
|
refresh_uri: str
|
|
@@ -618,7 +615,7 @@ class PluginConfig(yaml.YAMLObject):
|
|
|
618
615
|
#: :class:`~eodag.plugins.authentication.token_exchange.OIDCTokenExchangeAuth`
|
|
619
616
|
#: The full :class:`~eodag.plugins.authentication.openid_connect.OIDCAuthorizationCodeFlowAuth` plugin configuration
|
|
620
617
|
#: used to retrieve subject token
|
|
621
|
-
subject:
|
|
618
|
+
subject: dict[str, Any]
|
|
622
619
|
#: :class:`~eodag.plugins.authentication.token_exchange.OIDCTokenExchangeAuth`
|
|
623
620
|
#: Identifies the issuer of the `subject_token`
|
|
624
621
|
subject_issuer: str
|
|
@@ -640,21 +637,21 @@ class PluginConfig(yaml.YAMLObject):
|
|
|
640
637
|
return loader.construct_yaml_object(node, cls)
|
|
641
638
|
|
|
642
639
|
@classmethod
|
|
643
|
-
def from_mapping(cls, mapping:
|
|
640
|
+
def from_mapping(cls, mapping: dict[str, Any]) -> PluginConfig:
|
|
644
641
|
"""Build a :class:`~eodag.config.PluginConfig` from a mapping"""
|
|
645
642
|
c = cls()
|
|
646
643
|
c.__dict__.update(mapping)
|
|
647
644
|
return c
|
|
648
645
|
|
|
649
646
|
@staticmethod
|
|
650
|
-
def validate(config_keys:
|
|
647
|
+
def validate(config_keys: tuple[Any, ...]) -> None:
|
|
651
648
|
"""Validate a :class:`~eodag.config.PluginConfig`"""
|
|
652
649
|
if "type" not in config_keys:
|
|
653
650
|
raise ValidationError(
|
|
654
651
|
"A Plugin config must specify the Plugin it configures"
|
|
655
652
|
)
|
|
656
653
|
|
|
657
|
-
def update(self, mapping: Optional[
|
|
654
|
+
def update(self, mapping: Optional[dict[Any, Any]]) -> None:
|
|
658
655
|
"""Update the configuration parameters with values from `mapping`
|
|
659
656
|
|
|
660
657
|
:param mapping: The mapping from which to override configuration parameters
|
|
@@ -666,7 +663,7 @@ class PluginConfig(yaml.YAMLObject):
|
|
|
666
663
|
)
|
|
667
664
|
|
|
668
665
|
|
|
669
|
-
def load_default_config() ->
|
|
666
|
+
def load_default_config() -> dict[str, ProviderConfig]:
|
|
670
667
|
"""Load the providers configuration into a dictionary.
|
|
671
668
|
|
|
672
669
|
Load from eodag `resources/providers.yml` or `EODAG_PROVIDERS_CFG_FILE` environment
|
|
@@ -680,18 +677,18 @@ def load_default_config() -> Dict[str, ProviderConfig]:
|
|
|
680
677
|
return load_config(eodag_providers_cfg_file)
|
|
681
678
|
|
|
682
679
|
|
|
683
|
-
def load_config(config_path: str) ->
|
|
680
|
+
def load_config(config_path: str) -> dict[str, ProviderConfig]:
|
|
684
681
|
"""Load the providers configuration into a dictionary from a given file
|
|
685
682
|
|
|
686
683
|
:param config_path: The path to the provider config file
|
|
687
684
|
:returns: The default provider's configuration
|
|
688
685
|
"""
|
|
689
686
|
logger.debug("Loading configuration from %s", config_path)
|
|
690
|
-
config:
|
|
687
|
+
config: dict[str, ProviderConfig] = {}
|
|
691
688
|
try:
|
|
692
689
|
# Providers configs are stored in this file as separated yaml documents
|
|
693
690
|
# Load all of it
|
|
694
|
-
providers_configs:
|
|
691
|
+
providers_configs: list[ProviderConfig] = cached_yaml_load_all(config_path)
|
|
695
692
|
except yaml.parser.ParserError as e:
|
|
696
693
|
logger.error("Unable to load configuration")
|
|
697
694
|
raise e
|
|
@@ -714,7 +711,7 @@ def credentials_in_auth(auth_conf: PluginConfig) -> bool:
|
|
|
714
711
|
|
|
715
712
|
|
|
716
713
|
def share_credentials(
|
|
717
|
-
providers_config:
|
|
714
|
+
providers_config: dict[str, ProviderConfig],
|
|
718
715
|
) -> None:
|
|
719
716
|
"""Share credentials between plugins having the same matching criteria
|
|
720
717
|
|
|
@@ -758,7 +755,7 @@ def share_credentials(
|
|
|
758
755
|
|
|
759
756
|
def provider_config_init(
|
|
760
757
|
provider_config: ProviderConfig,
|
|
761
|
-
stac_search_default_conf: Optional[
|
|
758
|
+
stac_search_default_conf: Optional[dict[str, Any]] = None,
|
|
762
759
|
) -> None:
|
|
763
760
|
"""Applies some default values to provider config
|
|
764
761
|
|
|
@@ -796,7 +793,7 @@ def provider_config_init(
|
|
|
796
793
|
pass
|
|
797
794
|
|
|
798
795
|
|
|
799
|
-
def override_config_from_file(config:
|
|
796
|
+
def override_config_from_file(config: dict[str, Any], file_path: str) -> None:
|
|
800
797
|
"""Override a configuration with the values in a file
|
|
801
798
|
|
|
802
799
|
:param config: An eodag providers configuration dictionary
|
|
@@ -814,14 +811,14 @@ def override_config_from_file(config: Dict[str, Any], file_path: str) -> None:
|
|
|
814
811
|
override_config_from_mapping(config, config_in_file)
|
|
815
812
|
|
|
816
813
|
|
|
817
|
-
def override_config_from_env(config:
|
|
814
|
+
def override_config_from_env(config: dict[str, Any]) -> None:
|
|
818
815
|
"""Override a configuration with environment variables values
|
|
819
816
|
|
|
820
817
|
:param config: An eodag providers configuration dictionary
|
|
821
818
|
"""
|
|
822
819
|
|
|
823
820
|
def build_mapping_from_env(
|
|
824
|
-
env_var: str, env_value: str, mapping:
|
|
821
|
+
env_var: str, env_value: str, mapping: dict[str, Any]
|
|
825
822
|
) -> None:
|
|
826
823
|
"""Recursively build a dictionary from an environment variable.
|
|
827
824
|
|
|
@@ -872,7 +869,7 @@ def override_config_from_env(config: Dict[str, Any]) -> None:
|
|
|
872
869
|
new_map = mapping.setdefault(parts[0], {})
|
|
873
870
|
build_mapping_from_env("__".join(parts[1:]), env_value, new_map)
|
|
874
871
|
|
|
875
|
-
mapping_from_env:
|
|
872
|
+
mapping_from_env: dict[str, Any] = {}
|
|
876
873
|
for env_var in os.environ:
|
|
877
874
|
if env_var.startswith("EODAG__"):
|
|
878
875
|
build_mapping_from_env(
|
|
@@ -885,7 +882,7 @@ def override_config_from_env(config: Dict[str, Any]) -> None:
|
|
|
885
882
|
|
|
886
883
|
|
|
887
884
|
def override_config_from_mapping(
|
|
888
|
-
config:
|
|
885
|
+
config: dict[str, Any], mapping: dict[str, Any]
|
|
889
886
|
) -> None:
|
|
890
887
|
"""Override a configuration with the values in a mapping
|
|
891
888
|
|
|
@@ -923,7 +920,7 @@ def override_config_from_mapping(
|
|
|
923
920
|
)
|
|
924
921
|
|
|
925
922
|
# try overriding conf
|
|
926
|
-
old_conf: Optional[
|
|
923
|
+
old_conf: Optional[dict[str, Any]] = config.get(provider)
|
|
927
924
|
if old_conf is not None:
|
|
928
925
|
old_conf.update(new_conf)
|
|
929
926
|
else:
|
|
@@ -944,7 +941,7 @@ def override_config_from_mapping(
|
|
|
944
941
|
logger.debug(tb.format_exc())
|
|
945
942
|
|
|
946
943
|
|
|
947
|
-
def merge_configs(config:
|
|
944
|
+
def merge_configs(config: dict[str, Any], other_config: dict[str, Any]) -> None:
|
|
948
945
|
"""Override a configuration with the values of another configuration
|
|
949
946
|
|
|
950
947
|
:param config: An eodag providers configuration dictionary
|
|
@@ -976,7 +973,7 @@ def merge_configs(config: Dict[str, Any], other_config: Dict[str, Any]) -> None:
|
|
|
976
973
|
config[provider] = new_conf
|
|
977
974
|
|
|
978
975
|
|
|
979
|
-
def load_yml_config(yml_path: str) ->
|
|
976
|
+
def load_yml_config(yml_path: str) -> dict[Any, Any]:
|
|
980
977
|
"""Load a conf dictionary from given yml absolute path
|
|
981
978
|
|
|
982
979
|
:returns: The yml configuration file
|
|
@@ -985,7 +982,7 @@ def load_yml_config(yml_path: str) -> Dict[Any, Any]:
|
|
|
985
982
|
return dict_items_recursive_apply(config.source, string_to_jsonpath)
|
|
986
983
|
|
|
987
984
|
|
|
988
|
-
def load_stac_config() ->
|
|
985
|
+
def load_stac_config() -> dict[str, Any]:
|
|
989
986
|
"""Load the stac configuration into a dictionary
|
|
990
987
|
|
|
991
988
|
:returns: The stac configuration
|
|
@@ -995,7 +992,7 @@ def load_stac_config() -> Dict[str, Any]:
|
|
|
995
992
|
)
|
|
996
993
|
|
|
997
994
|
|
|
998
|
-
def load_stac_api_config() ->
|
|
995
|
+
def load_stac_api_config() -> dict[str, Any]:
|
|
999
996
|
"""Load the stac API configuration into a dictionary
|
|
1000
997
|
|
|
1001
998
|
:returns: The stac API configuration
|
|
@@ -1005,7 +1002,7 @@ def load_stac_api_config() -> Dict[str, Any]:
|
|
|
1005
1002
|
)
|
|
1006
1003
|
|
|
1007
1004
|
|
|
1008
|
-
def load_stac_provider_config() ->
|
|
1005
|
+
def load_stac_provider_config() -> dict[str, Any]:
|
|
1009
1006
|
"""Load the stac provider configuration into a dictionary
|
|
1010
1007
|
|
|
1011
1008
|
:returns: The stac provider configuration
|
|
@@ -1017,7 +1014,7 @@ def load_stac_provider_config() -> Dict[str, Any]:
|
|
|
1017
1014
|
|
|
1018
1015
|
def get_ext_product_types_conf(
|
|
1019
1016
|
conf_uri: str = EXT_PRODUCT_TYPES_CONF_URI,
|
|
1020
|
-
) ->
|
|
1017
|
+
) -> dict[str, Any]:
|
|
1021
1018
|
"""Read external product types conf
|
|
1022
1019
|
|
|
1023
1020
|
:param conf_uri: URI to local or remote configuration file
|
eodag/plugins/apis/base.py
CHANGED
eodag/plugins/apis/ecmwf.py
CHANGED
|
@@ -48,13 +48,14 @@ from eodag.utils.exceptions import AuthenticationError, DownloadError
|
|
|
48
48
|
from eodag.utils.logging import get_logging_verbose
|
|
49
49
|
|
|
50
50
|
if TYPE_CHECKING:
|
|
51
|
-
from typing import Any,
|
|
51
|
+
from typing import Any, Optional, Union
|
|
52
52
|
|
|
53
53
|
from requests.auth import AuthBase
|
|
54
54
|
|
|
55
55
|
from eodag.api.product import EOProduct
|
|
56
56
|
from eodag.api.search_result import SearchResult
|
|
57
57
|
from eodag.config import PluginConfig
|
|
58
|
+
from eodag.types import S3SessionKwargs
|
|
58
59
|
from eodag.types.download_args import DownloadConf
|
|
59
60
|
from eodag.utils import DownloadedCallback, ProgressCallback, Unpack
|
|
60
61
|
|
|
@@ -83,7 +84,7 @@ class EcmwfApi(Api, ECMWFSearch):
|
|
|
83
84
|
* :attr:`~eodag.config.PluginConfig.type` (``str``) (**mandatory**): EcmwfApi
|
|
84
85
|
* :attr:`~eodag.config.PluginConfig.auth_endpoint` (``str``) (**mandatory**): url of
|
|
85
86
|
the authentication endpoint of the ecmwf api
|
|
86
|
-
* :attr:`~eodag.config.PluginConfig.metadata_mapping` (``
|
|
87
|
+
* :attr:`~eodag.config.PluginConfig.metadata_mapping` (``dict[str, Union[str, list]]``): how
|
|
87
88
|
parameters should be mapped between the provider and eodag; If a string is given, this is
|
|
88
89
|
the mapping parameter returned by provider -> eodag parameter. If a list with 2 elements
|
|
89
90
|
is given, the first one is the mapping eodag parameter -> provider query parameters
|
|
@@ -104,7 +105,7 @@ class EcmwfApi(Api, ECMWFSearch):
|
|
|
104
105
|
self.config.__dict__.setdefault("pagination", {"next_page_query_obj": "{{}}"})
|
|
105
106
|
self.config.__dict__.setdefault("api_endpoint", "")
|
|
106
107
|
|
|
107
|
-
def do_search(self, *args: Any, **kwargs: Any) ->
|
|
108
|
+
def do_search(self, *args: Any, **kwargs: Any) -> list[dict[str, Any]]:
|
|
108
109
|
"""Should perform the actual search request."""
|
|
109
110
|
return [{}]
|
|
110
111
|
|
|
@@ -112,7 +113,7 @@ class EcmwfApi(Api, ECMWFSearch):
|
|
|
112
113
|
self,
|
|
113
114
|
prep: PreparedSearch = PreparedSearch(),
|
|
114
115
|
**kwargs: Any,
|
|
115
|
-
) ->
|
|
116
|
+
) -> tuple[list[EOProduct], Optional[int]]:
|
|
116
117
|
"""Build ready-to-download SearchResult"""
|
|
117
118
|
|
|
118
119
|
# check productType, dates, geometry, use defaults if not specified
|
|
@@ -145,7 +146,7 @@ class EcmwfApi(Api, ECMWFSearch):
|
|
|
145
146
|
|
|
146
147
|
return ECMWFSearch.query(self, prep, **kwargs)
|
|
147
148
|
|
|
148
|
-
def authenticate(self) ->
|
|
149
|
+
def authenticate(self) -> dict[str, Optional[str]]:
|
|
149
150
|
"""Check credentials and returns information needed for auth
|
|
150
151
|
|
|
151
152
|
:returns: {key, url, email} dictionary
|
|
@@ -176,7 +177,7 @@ class EcmwfApi(Api, ECMWFSearch):
|
|
|
176
177
|
def download(
|
|
177
178
|
self,
|
|
178
179
|
product: EOProduct,
|
|
179
|
-
auth: Optional[Union[AuthBase,
|
|
180
|
+
auth: Optional[Union[AuthBase, S3SessionKwargs]] = None,
|
|
180
181
|
progress_callback: Optional[ProgressCallback] = None,
|
|
181
182
|
wait: float = DEFAULT_DOWNLOAD_WAIT,
|
|
182
183
|
timeout: float = DEFAULT_DOWNLOAD_TIMEOUT,
|
|
@@ -265,13 +266,13 @@ class EcmwfApi(Api, ECMWFSearch):
|
|
|
265
266
|
def download_all(
|
|
266
267
|
self,
|
|
267
268
|
products: SearchResult,
|
|
268
|
-
auth: Optional[Union[AuthBase,
|
|
269
|
+
auth: Optional[Union[AuthBase, S3SessionKwargs]] = None,
|
|
269
270
|
downloaded_callback: Optional[DownloadedCallback] = None,
|
|
270
271
|
progress_callback: Optional[ProgressCallback] = None,
|
|
271
272
|
wait: float = DEFAULT_DOWNLOAD_WAIT,
|
|
272
273
|
timeout: float = DEFAULT_DOWNLOAD_TIMEOUT,
|
|
273
274
|
**kwargs: Unpack[DownloadConf],
|
|
274
|
-
) ->
|
|
275
|
+
) -> list[str]:
|
|
275
276
|
"""
|
|
276
277
|
Download all using parent (base plugin) method
|
|
277
278
|
"""
|
|
@@ -291,7 +292,7 @@ class EcmwfApi(Api, ECMWFSearch):
|
|
|
291
292
|
|
|
292
293
|
def discover_queryables(
|
|
293
294
|
self, **kwargs: Any
|
|
294
|
-
) -> Optional[
|
|
295
|
+
) -> Optional[dict[str, Annotated[Any, FieldInfo]]]:
|
|
295
296
|
"""Fetch queryables list from provider using metadata mapping
|
|
296
297
|
|
|
297
298
|
:param kwargs: additional filters for queryables (`productType` and other search
|