eodag 3.1.0b1__py3-none-any.whl → 3.2.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- eodag/api/core.py +69 -63
- eodag/api/product/_assets.py +49 -13
- eodag/api/product/_product.py +41 -30
- eodag/api/product/drivers/__init__.py +81 -4
- eodag/api/product/drivers/base.py +65 -4
- eodag/api/product/drivers/generic.py +65 -0
- eodag/api/product/drivers/sentinel1.py +97 -0
- eodag/api/product/drivers/sentinel2.py +95 -0
- eodag/api/product/metadata_mapping.py +85 -79
- eodag/api/search_result.py +13 -23
- eodag/cli.py +4 -4
- eodag/config.py +77 -80
- eodag/plugins/apis/base.py +1 -1
- eodag/plugins/apis/ecmwf.py +12 -15
- eodag/plugins/apis/usgs.py +12 -11
- eodag/plugins/authentication/aws_auth.py +16 -13
- eodag/plugins/authentication/base.py +5 -3
- eodag/plugins/authentication/header.py +3 -3
- eodag/plugins/authentication/keycloak.py +4 -4
- eodag/plugins/authentication/oauth.py +7 -3
- eodag/plugins/authentication/openid_connect.py +20 -14
- eodag/plugins/authentication/sas_auth.py +4 -4
- eodag/plugins/authentication/token.py +7 -7
- eodag/plugins/authentication/token_exchange.py +1 -1
- eodag/plugins/base.py +4 -4
- eodag/plugins/crunch/base.py +4 -4
- eodag/plugins/crunch/filter_date.py +4 -4
- eodag/plugins/crunch/filter_latest_intersect.py +6 -6
- eodag/plugins/crunch/filter_latest_tpl_name.py +7 -7
- eodag/plugins/crunch/filter_overlap.py +4 -4
- eodag/plugins/crunch/filter_property.py +4 -4
- eodag/plugins/download/aws.py +137 -77
- eodag/plugins/download/base.py +8 -17
- eodag/plugins/download/creodias_s3.py +2 -2
- eodag/plugins/download/http.py +30 -32
- eodag/plugins/download/s3rest.py +5 -4
- eodag/plugins/manager.py +10 -20
- eodag/plugins/search/__init__.py +6 -5
- eodag/plugins/search/base.py +38 -42
- eodag/plugins/search/build_search_result.py +286 -336
- eodag/plugins/search/cop_marine.py +22 -12
- eodag/plugins/search/creodias_s3.py +8 -78
- eodag/plugins/search/csw.py +11 -11
- eodag/plugins/search/data_request_search.py +19 -18
- eodag/plugins/search/qssearch.py +84 -151
- eodag/plugins/search/stac_list_assets.py +85 -0
- eodag/plugins/search/static_stac_search.py +4 -4
- eodag/resources/ext_product_types.json +1 -1
- eodag/resources/product_types.yml +848 -398
- eodag/resources/providers.yml +1038 -1115
- eodag/resources/stac_api.yml +2 -2
- eodag/resources/user_conf_template.yml +10 -9
- eodag/rest/cache.py +2 -2
- eodag/rest/config.py +3 -3
- eodag/rest/core.py +24 -24
- eodag/rest/errors.py +5 -5
- eodag/rest/server.py +3 -11
- eodag/rest/stac.py +41 -38
- eodag/rest/types/collections_search.py +3 -3
- eodag/rest/types/eodag_search.py +23 -23
- eodag/rest/types/queryables.py +40 -28
- eodag/rest/types/stac_search.py +15 -25
- eodag/rest/utils/__init__.py +11 -21
- eodag/rest/utils/cql_evaluate.py +6 -6
- eodag/rest/utils/rfc3339.py +2 -2
- eodag/types/__init__.py +97 -29
- eodag/types/bbox.py +2 -2
- eodag/types/download_args.py +2 -2
- eodag/types/queryables.py +5 -2
- eodag/types/search_args.py +4 -4
- eodag/types/whoosh.py +1 -3
- eodag/utils/__init__.py +82 -41
- eodag/utils/exceptions.py +2 -2
- eodag/utils/import_system.py +2 -2
- eodag/utils/requests.py +2 -2
- eodag/utils/rest.py +2 -2
- eodag/utils/s3.py +231 -0
- eodag/utils/stac_reader.py +10 -10
- {eodag-3.1.0b1.dist-info → eodag-3.2.0.dist-info}/METADATA +12 -10
- eodag-3.2.0.dist-info/RECORD +113 -0
- {eodag-3.1.0b1.dist-info → eodag-3.2.0.dist-info}/WHEEL +1 -1
- {eodag-3.1.0b1.dist-info → eodag-3.2.0.dist-info}/entry_points.txt +1 -0
- eodag-3.1.0b1.dist-info/RECORD +0 -108
- {eodag-3.1.0b1.dist-info → eodag-3.2.0.dist-info/licenses}/LICENSE +0 -0
- {eodag-3.1.0b1.dist-info → eodag-3.2.0.dist-info}/top_level.txt +0 -0
eodag/config.py
CHANGED
|
@@ -20,17 +20,15 @@ from __future__ import annotations
|
|
|
20
20
|
import logging
|
|
21
21
|
import os
|
|
22
22
|
import tempfile
|
|
23
|
+
from importlib.resources import files as res_files
|
|
23
24
|
from inspect import isclass
|
|
24
25
|
from typing import (
|
|
25
26
|
Annotated,
|
|
26
27
|
Any,
|
|
27
|
-
Dict,
|
|
28
28
|
ItemsView,
|
|
29
29
|
Iterator,
|
|
30
|
-
List,
|
|
31
30
|
Literal,
|
|
32
31
|
Optional,
|
|
33
|
-
Tuple,
|
|
34
32
|
TypedDict,
|
|
35
33
|
Union,
|
|
36
34
|
ValuesView,
|
|
@@ -44,7 +42,6 @@ import yaml.constructor
|
|
|
44
42
|
import yaml.parser
|
|
45
43
|
from annotated_types import Gt
|
|
46
44
|
from jsonpath_ng import JSONPath
|
|
47
|
-
from pkg_resources import resource_filename
|
|
48
45
|
|
|
49
46
|
from eodag.api.product.metadata_mapping import mtd_cfg_as_conversion_and_querypath
|
|
50
47
|
from eodag.utils import (
|
|
@@ -79,7 +76,7 @@ class SimpleYamlProxyConfig:
|
|
|
79
76
|
|
|
80
77
|
def __init__(self, conf_file_path: str) -> None:
|
|
81
78
|
try:
|
|
82
|
-
self.source:
|
|
79
|
+
self.source: dict[str, Any] = cached_yaml_load(conf_file_path)
|
|
83
80
|
except yaml.parser.ParserError as e:
|
|
84
81
|
print("Unable to load user configuration file")
|
|
85
82
|
raise e
|
|
@@ -127,12 +124,12 @@ class ProviderConfig(yaml.YAMLObject):
|
|
|
127
124
|
name: str
|
|
128
125
|
group: str
|
|
129
126
|
priority: int = 0 # Set default priority to 0
|
|
130
|
-
roles:
|
|
127
|
+
roles: list[str]
|
|
131
128
|
description: str
|
|
132
129
|
url: str
|
|
133
130
|
api: PluginConfig
|
|
134
131
|
search: PluginConfig
|
|
135
|
-
products:
|
|
132
|
+
products: dict[str, Any]
|
|
136
133
|
download: PluginConfig
|
|
137
134
|
auth: PluginConfig
|
|
138
135
|
search_auth: PluginConfig
|
|
@@ -154,7 +151,7 @@ class ProviderConfig(yaml.YAMLObject):
|
|
|
154
151
|
return loader.construct_yaml_object(node, cls)
|
|
155
152
|
|
|
156
153
|
@classmethod
|
|
157
|
-
def from_mapping(cls, mapping:
|
|
154
|
+
def from_mapping(cls, mapping: dict[str, Any]) -> ProviderConfig:
|
|
158
155
|
"""Build a :class:`~eodag.config.ProviderConfig` from a mapping"""
|
|
159
156
|
cls.validate(mapping)
|
|
160
157
|
for key in PLUGINS_TOPICS_KEYS:
|
|
@@ -165,7 +162,7 @@ class ProviderConfig(yaml.YAMLObject):
|
|
|
165
162
|
return c
|
|
166
163
|
|
|
167
164
|
@staticmethod
|
|
168
|
-
def validate(config_keys: Union[
|
|
165
|
+
def validate(config_keys: Union[tuple[str, ...], dict[str, Any]]) -> None:
|
|
169
166
|
"""Validate a :class:`~eodag.config.ProviderConfig`
|
|
170
167
|
|
|
171
168
|
:param config_keys: The configurations keys to validate
|
|
@@ -181,7 +178,7 @@ class ProviderConfig(yaml.YAMLObject):
|
|
|
181
178
|
"type of plugin"
|
|
182
179
|
)
|
|
183
180
|
|
|
184
|
-
def update(self, mapping: Optional[
|
|
181
|
+
def update(self, mapping: Optional[dict[str, Any]]) -> None:
|
|
185
182
|
"""Update the configuration parameters with values from `mapping`
|
|
186
183
|
|
|
187
184
|
:param mapping: The mapping from which to override configuration parameters
|
|
@@ -197,7 +194,7 @@ class ProviderConfig(yaml.YAMLObject):
|
|
|
197
194
|
},
|
|
198
195
|
)
|
|
199
196
|
for key in PLUGINS_TOPICS_KEYS:
|
|
200
|
-
current_value: Optional[
|
|
197
|
+
current_value: Optional[dict[str, Any]] = getattr(self, key, None)
|
|
201
198
|
mapping_value = mapping.get(key, {})
|
|
202
199
|
if current_value is not None:
|
|
203
200
|
current_value.update(mapping_value)
|
|
@@ -240,14 +237,14 @@ class PluginConfig(yaml.YAMLObject):
|
|
|
240
237
|
"""Configuration for sort during search"""
|
|
241
238
|
|
|
242
239
|
#: Default sort settings
|
|
243
|
-
sort_by_default:
|
|
240
|
+
sort_by_default: list[tuple[str, str]]
|
|
244
241
|
#: F-string template to add to :attr:`~eodag.config.PluginConfig.Pagination.next_page_url_tpl` to sort search
|
|
245
242
|
#: results
|
|
246
243
|
sort_by_tpl: str
|
|
247
244
|
#: Mapping between eodag and provider query parameters used for sort
|
|
248
|
-
sort_param_mapping:
|
|
245
|
+
sort_param_mapping: dict[str, str]
|
|
249
246
|
#: Mapping between eodag and provider sort-order parameters
|
|
250
|
-
sort_order_mapping:
|
|
247
|
+
sort_order_mapping: dict[Literal["ascending", "descending"], str]
|
|
251
248
|
#: Maximum number of allowed sort parameters per request
|
|
252
249
|
max_sort_params: Annotated[int, Gt(0)]
|
|
253
250
|
|
|
@@ -271,7 +268,7 @@ class PluginConfig(yaml.YAMLObject):
|
|
|
271
268
|
#: HTTP method used to fetch product types
|
|
272
269
|
fetch_method: str
|
|
273
270
|
#: Request body to fetch product types using POST method
|
|
274
|
-
fetch_body:
|
|
271
|
+
fetch_body: dict[str, Any]
|
|
275
272
|
#: Maximum number of connections for concurrent HTTP requests
|
|
276
273
|
max_connections: int
|
|
277
274
|
#: The f-string template for pagination requests.
|
|
@@ -286,17 +283,17 @@ class PluginConfig(yaml.YAMLObject):
|
|
|
286
283
|
generic_product_type_id: str
|
|
287
284
|
#: Mapping for product type metadata (e.g. ``abstract``, ``licence``) which can be parsed from the provider
|
|
288
285
|
#: result
|
|
289
|
-
generic_product_type_parsable_metadata:
|
|
286
|
+
generic_product_type_parsable_metadata: dict[str, str]
|
|
290
287
|
#: Mapping for product type properties which can be parsed from the result and are not product type metadata
|
|
291
|
-
generic_product_type_parsable_properties:
|
|
288
|
+
generic_product_type_parsable_properties: dict[str, str]
|
|
292
289
|
#: Mapping for product type properties which cannot be parsed from the result and are not product type metadata
|
|
293
|
-
generic_product_type_unparsable_properties:
|
|
290
|
+
generic_product_type_unparsable_properties: dict[str, str]
|
|
294
291
|
#: URL to fetch data for a single collection
|
|
295
292
|
single_collection_fetch_url: str
|
|
296
293
|
#: Query string to be added to the fetch_url to filter for a collection
|
|
297
294
|
single_collection_fetch_qs: str
|
|
298
295
|
#: Mapping for product type metadata returned by the endpoint given in single_collection_fetch_url
|
|
299
|
-
single_product_type_parsable_metadata:
|
|
296
|
+
single_product_type_parsable_metadata: dict[str, str]
|
|
300
297
|
|
|
301
298
|
class DiscoverQueryables(TypedDict, total=False):
|
|
302
299
|
"""Configuration for queryables discovery"""
|
|
@@ -318,7 +315,7 @@ class PluginConfig(yaml.YAMLObject):
|
|
|
318
315
|
"""Configuration for order on-response during download"""
|
|
319
316
|
|
|
320
317
|
#: Parameters metadata-mapping to apply to the order response
|
|
321
|
-
metadata_mapping:
|
|
318
|
+
metadata_mapping: dict[str, Union[str, list[str]]]
|
|
322
319
|
|
|
323
320
|
class OrderStatusSuccess(TypedDict):
|
|
324
321
|
"""
|
|
@@ -351,7 +348,7 @@ class PluginConfig(yaml.YAMLObject):
|
|
|
351
348
|
#: Request HTTP method
|
|
352
349
|
method: str
|
|
353
350
|
#: Request hearders
|
|
354
|
-
headers:
|
|
351
|
+
headers: dict[str, Any]
|
|
355
352
|
|
|
356
353
|
class OrderStatusOnSuccess(TypedDict, total=False):
|
|
357
354
|
"""Configuration for order status on-success during download"""
|
|
@@ -363,7 +360,7 @@ class PluginConfig(yaml.YAMLObject):
|
|
|
363
360
|
#: Key in the success response that gives access to the result
|
|
364
361
|
results_entry: str
|
|
365
362
|
#: Metadata-mapping to apply to the success status result
|
|
366
|
-
metadata_mapping:
|
|
363
|
+
metadata_mapping: dict[str, Union[str, list[str]]]
|
|
367
364
|
|
|
368
365
|
class OrderStatus(TypedDict, total=False):
|
|
369
366
|
"""Configuration for order status during download"""
|
|
@@ -371,11 +368,11 @@ class PluginConfig(yaml.YAMLObject):
|
|
|
371
368
|
#: Order status request configuration
|
|
372
369
|
request: PluginConfig.OrderStatusRequest
|
|
373
370
|
#: Metadata-mapping used to parse order status response
|
|
374
|
-
metadata_mapping:
|
|
371
|
+
metadata_mapping: dict[str, Union[str, list[str]]]
|
|
375
372
|
#: Configuration to identify order status success during download
|
|
376
373
|
success: PluginConfig.OrderStatusSuccess
|
|
377
374
|
#: Part of the order status response that tells there is an error
|
|
378
|
-
error:
|
|
375
|
+
error: dict[str, Any]
|
|
379
376
|
#: Configuration to identify order status ordered during download
|
|
380
377
|
ordered: PluginConfig.OrderStatusOrdered
|
|
381
378
|
#: Configuration for order status on-success during download
|
|
@@ -400,7 +397,7 @@ class PluginConfig(yaml.YAMLObject):
|
|
|
400
397
|
#: :class:`~eodag.plugins.base.PluginTopic` Default s3 bucket
|
|
401
398
|
s3_bucket: str
|
|
402
399
|
#: :class:`~eodag.plugins.base.PluginTopic` Authentication error codes
|
|
403
|
-
auth_error_code: Union[int,
|
|
400
|
+
auth_error_code: Union[int, list[int]]
|
|
404
401
|
#: :class:`~eodag.plugins.base.PluginTopic` Time to wait until request timeout in seconds
|
|
405
402
|
timeout: float
|
|
406
403
|
#: :class:`~eodag.plugins.base.PluginTopic` :class:`urllib3.util.Retry` ``total`` parameter,
|
|
@@ -411,13 +408,13 @@ class PluginConfig(yaml.YAMLObject):
|
|
|
411
408
|
retry_backoff_factor: int
|
|
412
409
|
#: :class:`~eodag.plugins.base.PluginTopic` :class:`urllib3.util.Retry` ``status_forcelist`` parameter,
|
|
413
410
|
#: list of integer HTTP status codes that we should force a retry on
|
|
414
|
-
retry_status_forcelist:
|
|
411
|
+
retry_status_forcelist: list[int]
|
|
415
412
|
|
|
416
413
|
# search & api -----------------------------------------------------------------------------------------------------
|
|
417
414
|
# copied from ProviderConfig in PluginManager.get_search_plugins()
|
|
418
415
|
priority: int
|
|
419
416
|
# per product type metadata-mapping, set in core._prepare_search
|
|
420
|
-
product_type_config:
|
|
417
|
+
product_type_config: dict[str, Any]
|
|
421
418
|
|
|
422
419
|
#: :class:`~eodag.plugins.search.base.Search` Plugin API endpoint
|
|
423
420
|
api_endpoint: str
|
|
@@ -439,15 +436,18 @@ class PluginConfig(yaml.YAMLObject):
|
|
|
439
436
|
#: :class:`~eodag.plugins.search.base.Search` Configuration for the queryables auto-discovery
|
|
440
437
|
discover_queryables: PluginConfig.DiscoverQueryables
|
|
441
438
|
#: :class:`~eodag.plugins.search.base.Search` The mapping between eodag metadata and the plugin specific metadata
|
|
442
|
-
metadata_mapping:
|
|
439
|
+
metadata_mapping: dict[str, Union[str, list[str]]]
|
|
443
440
|
#: :class:`~eodag.plugins.search.base.Search` Parameters to remove from queryables
|
|
444
|
-
remove_from_queryables:
|
|
441
|
+
remove_from_queryables: list[str]
|
|
445
442
|
#: :class:`~eodag.plugins.search.base.Search` Parameters to be passed as is in the search url query string
|
|
446
|
-
literal_search_params:
|
|
443
|
+
literal_search_params: dict[str, str]
|
|
447
444
|
#: :class:`~eodag.plugins.search.qssearch.QueryStringSearch` Characters that should not be quoted in the url params
|
|
448
|
-
dont_quote:
|
|
445
|
+
dont_quote: list[str]
|
|
446
|
+
#: :class:`~eodag.plugins.search.qssearch.QueryStringSearch` Guess assets keys using their ``href``.
|
|
447
|
+
#: Use their original key if ``False``
|
|
448
|
+
asset_key_from_href: bool
|
|
449
449
|
#: :class:`~eodag.plugins.search.qssearch.ODataV4Search` Dict describing free text search request build
|
|
450
|
-
free_text_search_operations:
|
|
450
|
+
free_text_search_operations: dict[str, Any]
|
|
451
451
|
#: :class:`~eodag.plugins.search.qssearch.ODataV4Search` Set to ``True`` if the metadata is not given in the search
|
|
452
452
|
#: result and a two step search has to be performed
|
|
453
453
|
per_product_metadata_query: bool
|
|
@@ -464,11 +464,11 @@ class PluginConfig(yaml.YAMLObject):
|
|
|
464
464
|
#: if date parameters are mandatory in the request
|
|
465
465
|
dates_required: bool
|
|
466
466
|
#: :class:`~eodag.plugins.search.csw.CSWSearch` Search definition dictionary
|
|
467
|
-
search_definition:
|
|
467
|
+
search_definition: dict[str, Any]
|
|
468
468
|
#: :class:`~eodag.plugins.search.qssearch.PostJsonSearch` Whether to merge responses or not (`aws_eos` specific)
|
|
469
469
|
merge_responses: bool
|
|
470
470
|
#: :class:`~eodag.plugins.search.qssearch.PostJsonSearch` Collections names (`aws_eos` specific)
|
|
471
|
-
collection:
|
|
471
|
+
collection: list[str]
|
|
472
472
|
#: :class:`~eodag.plugins.search.static_stac_search.StaticStacSearch`
|
|
473
473
|
#: Maximum number of connections for concurrent HTTP requests
|
|
474
474
|
max_connections: int
|
|
@@ -477,7 +477,7 @@ class PluginConfig(yaml.YAMLObject):
|
|
|
477
477
|
end_date_excluded: bool
|
|
478
478
|
#: :class:`~eodag.plugins.search.build_search_result.ECMWFSearch`
|
|
479
479
|
#: List of parameters used to parse metadata but that must not be included to the query
|
|
480
|
-
remove_from_query:
|
|
480
|
+
remove_from_query: list[str]
|
|
481
481
|
#: :class:`~eodag.plugins.search.csw.CSWSearch`
|
|
482
482
|
#: OGC Catalogue Service version
|
|
483
483
|
version: str
|
|
@@ -501,13 +501,13 @@ class PluginConfig(yaml.YAMLObject):
|
|
|
501
501
|
#: :class:`~eodag.plugins.download.base.Download` Whether ignore assets and download using ``downloadLink`` or not
|
|
502
502
|
ignore_assets: bool
|
|
503
503
|
#: :class:`~eodag.plugins.download.base.Download` Product type specific configuration
|
|
504
|
-
products:
|
|
504
|
+
products: dict[str, dict[str, Any]]
|
|
505
505
|
#: :class:`~eodag.plugins.download.http.HTTPDownload` Whether the product has to be ordered to download it or not
|
|
506
506
|
order_enabled: bool
|
|
507
507
|
#: :class:`~eodag.plugins.download.http.HTTPDownload` HTTP request method for the order request
|
|
508
508
|
order_method: str
|
|
509
509
|
#: :class:`~eodag.plugins.download.http.HTTPDownload` Headers to be added to the order request
|
|
510
|
-
order_headers:
|
|
510
|
+
order_headers: dict[str, str]
|
|
511
511
|
#: :class:`~eodag.plugins.download.http.HTTPDownload`
|
|
512
512
|
#: Dictionary containing the key :attr:`~eodag.config.PluginConfig.metadata_mapping` which can be used to add new
|
|
513
513
|
#: product properties based on the data in response to the order request
|
|
@@ -518,7 +518,7 @@ class PluginConfig(yaml.YAMLObject):
|
|
|
518
518
|
#: Do not authenticate the download request but only the order and order status ones
|
|
519
519
|
no_auth_download: bool
|
|
520
520
|
#: :class:`~eodag.plugins.download.http.HTTPDownload` Parameters to be added to the query params of the request
|
|
521
|
-
dl_url_params:
|
|
521
|
+
dl_url_params: dict[str, str]
|
|
522
522
|
#: :class:`~eodag.plugins.download.s3rest.S3RestDownload`
|
|
523
523
|
#: At which level of the path part of the url the bucket can be found
|
|
524
524
|
bucket_path_level: int
|
|
@@ -529,15 +529,15 @@ class PluginConfig(yaml.YAMLObject):
|
|
|
529
529
|
|
|
530
530
|
# auth -------------------------------------------------------------------------------------------------------------
|
|
531
531
|
#: :class:`~eodag.plugins.authentication.base.Authentication` Authentication credentials dictionary
|
|
532
|
-
credentials:
|
|
532
|
+
credentials: dict[str, str]
|
|
533
533
|
#: :class:`~eodag.plugins.authentication.base.Authentication` Authentication URL
|
|
534
534
|
auth_uri: str
|
|
535
535
|
#: :class:`~eodag.plugins.authentication.base.Authentication`
|
|
536
536
|
#: Dictionary containing all keys/value pairs that should be added to the headers
|
|
537
|
-
headers:
|
|
537
|
+
headers: dict[str, str]
|
|
538
538
|
#: :class:`~eodag.plugins.authentication.base.Authentication`
|
|
539
539
|
#: Dictionary containing all keys/value pairs that should be added to the headers for token retrieve only
|
|
540
|
-
retrieve_headers:
|
|
540
|
+
retrieve_headers: dict[str, str]
|
|
541
541
|
#: :class:`~eodag.plugins.authentication.base.Authentication`
|
|
542
542
|
#: The key pointing to the token in the response from the token server
|
|
543
543
|
token_key: str
|
|
@@ -549,7 +549,7 @@ class PluginConfig(yaml.YAMLObject):
|
|
|
549
549
|
matching_url: str
|
|
550
550
|
#: :class:`~eodag.plugins.authentication.base.Authentication` Part of the search or download plugin configuration
|
|
551
551
|
#: that needs authentication
|
|
552
|
-
matching_conf:
|
|
552
|
+
matching_conf: dict[str, Any]
|
|
553
553
|
#: :class:`~eodag.plugins.authentication.openid_connect.OIDCRefreshTokenBase`
|
|
554
554
|
#: How the token should be used in the request
|
|
555
555
|
token_provision: str
|
|
@@ -561,7 +561,7 @@ class PluginConfig(yaml.YAMLObject):
|
|
|
561
561
|
#: The OIDC provider's ``.well-known/openid-configuration`` url.
|
|
562
562
|
oidc_config_url: str
|
|
563
563
|
#: :class:`~eodag.plugins.authentication.openid_connect.OIDCRefreshTokenBase` The OIDC token audiences
|
|
564
|
-
allowed_audiences:
|
|
564
|
+
allowed_audiences: list[str]
|
|
565
565
|
#: :class:`~eodag.plugins.authentication.openid_connect.OIDCAuthorizationCodeFlowAuth`
|
|
566
566
|
#: Whether a user consent is needed during the authentication or not
|
|
567
567
|
user_consent_needed: str
|
|
@@ -585,16 +585,16 @@ class PluginConfig(yaml.YAMLObject):
|
|
|
585
585
|
user_consent_form_xpath: str
|
|
586
586
|
#: :class:`~eodag.plugins.authentication.openid_connect.OIDCAuthorizationCodeFlowAuth`
|
|
587
587
|
#: The data that will be passed with the POST request on the form 'action' URL
|
|
588
|
-
user_consent_form_data:
|
|
588
|
+
user_consent_form_data: dict[str, str]
|
|
589
589
|
#: :class:`~eodag.plugins.authentication.openid_connect.OIDCAuthorizationCodeFlowAuth`
|
|
590
590
|
#: Additional data to be passed to the login POST request
|
|
591
|
-
additional_login_form_data:
|
|
591
|
+
additional_login_form_data: dict[str, str]
|
|
592
592
|
#: :class:`~eodag.plugins.authentication.openid_connect.OIDCAuthorizationCodeFlowAuth`
|
|
593
593
|
#: Key/value pairs of patterns/messages used for Authentication errors
|
|
594
|
-
exchange_url_error_pattern:
|
|
594
|
+
exchange_url_error_pattern: dict[str, str]
|
|
595
595
|
#: :class:`~eodag.plugins.authentication.openid_connect.OIDCAuthorizationCodeFlowAuth`
|
|
596
596
|
#: A mapping between OIDC url query string and token handler query string params
|
|
597
|
-
token_exchange_params:
|
|
597
|
+
token_exchange_params: dict[str, str]
|
|
598
598
|
#: :class:`~eodag.plugins.authentication.openid_connect.OIDCAuthorizationCodeFlowAuth`
|
|
599
599
|
#: Refers to the name of the query param to be used in the query request
|
|
600
600
|
token_qs_key: str
|
|
@@ -608,7 +608,7 @@ class PluginConfig(yaml.YAMLObject):
|
|
|
608
608
|
signed_url_key: str
|
|
609
609
|
#: :class:`~eodag.plugins.authentication.token.TokenAuth`
|
|
610
610
|
#: Credentials json structure if they should be sent as POST data
|
|
611
|
-
req_data:
|
|
611
|
+
req_data: dict[str, Any]
|
|
612
612
|
#: :class:`~eodag.plugins.authentication.token.TokenAuth`
|
|
613
613
|
#: URL used to fetch the access token with a refresh token
|
|
614
614
|
refresh_uri: str
|
|
@@ -618,7 +618,7 @@ class PluginConfig(yaml.YAMLObject):
|
|
|
618
618
|
#: :class:`~eodag.plugins.authentication.token_exchange.OIDCTokenExchangeAuth`
|
|
619
619
|
#: The full :class:`~eodag.plugins.authentication.openid_connect.OIDCAuthorizationCodeFlowAuth` plugin configuration
|
|
620
620
|
#: used to retrieve subject token
|
|
621
|
-
subject:
|
|
621
|
+
subject: dict[str, Any]
|
|
622
622
|
#: :class:`~eodag.plugins.authentication.token_exchange.OIDCTokenExchangeAuth`
|
|
623
623
|
#: Identifies the issuer of the `subject_token`
|
|
624
624
|
subject_issuer: str
|
|
@@ -640,21 +640,21 @@ class PluginConfig(yaml.YAMLObject):
|
|
|
640
640
|
return loader.construct_yaml_object(node, cls)
|
|
641
641
|
|
|
642
642
|
@classmethod
|
|
643
|
-
def from_mapping(cls, mapping:
|
|
643
|
+
def from_mapping(cls, mapping: dict[str, Any]) -> PluginConfig:
|
|
644
644
|
"""Build a :class:`~eodag.config.PluginConfig` from a mapping"""
|
|
645
645
|
c = cls()
|
|
646
646
|
c.__dict__.update(mapping)
|
|
647
647
|
return c
|
|
648
648
|
|
|
649
649
|
@staticmethod
|
|
650
|
-
def validate(config_keys:
|
|
650
|
+
def validate(config_keys: tuple[Any, ...]) -> None:
|
|
651
651
|
"""Validate a :class:`~eodag.config.PluginConfig`"""
|
|
652
652
|
if "type" not in config_keys:
|
|
653
653
|
raise ValidationError(
|
|
654
654
|
"A Plugin config must specify the Plugin it configures"
|
|
655
655
|
)
|
|
656
656
|
|
|
657
|
-
def update(self, mapping: Optional[
|
|
657
|
+
def update(self, mapping: Optional[dict[Any, Any]]) -> None:
|
|
658
658
|
"""Update the configuration parameters with values from `mapping`
|
|
659
659
|
|
|
660
660
|
:param mapping: The mapping from which to override configuration parameters
|
|
@@ -666,7 +666,7 @@ class PluginConfig(yaml.YAMLObject):
|
|
|
666
666
|
)
|
|
667
667
|
|
|
668
668
|
|
|
669
|
-
def load_default_config() ->
|
|
669
|
+
def load_default_config() -> dict[str, ProviderConfig]:
|
|
670
670
|
"""Load the providers configuration into a dictionary.
|
|
671
671
|
|
|
672
672
|
Load from eodag `resources/providers.yml` or `EODAG_PROVIDERS_CFG_FILE` environment
|
|
@@ -674,24 +674,24 @@ def load_default_config() -> Dict[str, ProviderConfig]:
|
|
|
674
674
|
|
|
675
675
|
:returns: The default provider's configuration
|
|
676
676
|
"""
|
|
677
|
-
eodag_providers_cfg_file = os.getenv(
|
|
678
|
-
"
|
|
679
|
-
)
|
|
677
|
+
eodag_providers_cfg_file = os.getenv("EODAG_PROVIDERS_CFG_FILE") or str(
|
|
678
|
+
res_files("eodag") / "resources" / "providers.yml"
|
|
679
|
+
)
|
|
680
680
|
return load_config(eodag_providers_cfg_file)
|
|
681
681
|
|
|
682
682
|
|
|
683
|
-
def load_config(config_path: str) ->
|
|
683
|
+
def load_config(config_path: str) -> dict[str, ProviderConfig]:
|
|
684
684
|
"""Load the providers configuration into a dictionary from a given file
|
|
685
685
|
|
|
686
686
|
:param config_path: The path to the provider config file
|
|
687
687
|
:returns: The default provider's configuration
|
|
688
688
|
"""
|
|
689
689
|
logger.debug("Loading configuration from %s", config_path)
|
|
690
|
-
config:
|
|
690
|
+
config: dict[str, ProviderConfig] = {}
|
|
691
691
|
try:
|
|
692
692
|
# Providers configs are stored in this file as separated yaml documents
|
|
693
693
|
# Load all of it
|
|
694
|
-
providers_configs:
|
|
694
|
+
providers_configs: list[ProviderConfig] = cached_yaml_load_all(config_path)
|
|
695
695
|
except yaml.parser.ParserError as e:
|
|
696
696
|
logger.error("Unable to load configuration")
|
|
697
697
|
raise e
|
|
@@ -714,7 +714,7 @@ def credentials_in_auth(auth_conf: PluginConfig) -> bool:
|
|
|
714
714
|
|
|
715
715
|
|
|
716
716
|
def share_credentials(
|
|
717
|
-
providers_config:
|
|
717
|
+
providers_config: dict[str, ProviderConfig],
|
|
718
718
|
) -> None:
|
|
719
719
|
"""Share credentials between plugins having the same matching criteria
|
|
720
720
|
|
|
@@ -758,7 +758,7 @@ def share_credentials(
|
|
|
758
758
|
|
|
759
759
|
def provider_config_init(
|
|
760
760
|
provider_config: ProviderConfig,
|
|
761
|
-
stac_search_default_conf: Optional[
|
|
761
|
+
stac_search_default_conf: Optional[dict[str, Any]] = None,
|
|
762
762
|
) -> None:
|
|
763
763
|
"""Applies some default values to provider config
|
|
764
764
|
|
|
@@ -782,6 +782,7 @@ def provider_config_init(
|
|
|
782
782
|
and provider_config.search.type
|
|
783
783
|
in [
|
|
784
784
|
"StacSearch",
|
|
785
|
+
"StacListAssets",
|
|
785
786
|
"StaticStacSearch",
|
|
786
787
|
]
|
|
787
788
|
):
|
|
@@ -796,7 +797,7 @@ def provider_config_init(
|
|
|
796
797
|
pass
|
|
797
798
|
|
|
798
799
|
|
|
799
|
-
def override_config_from_file(config:
|
|
800
|
+
def override_config_from_file(config: dict[str, Any], file_path: str) -> None:
|
|
800
801
|
"""Override a configuration with the values in a file
|
|
801
802
|
|
|
802
803
|
:param config: An eodag providers configuration dictionary
|
|
@@ -814,14 +815,14 @@ def override_config_from_file(config: Dict[str, Any], file_path: str) -> None:
|
|
|
814
815
|
override_config_from_mapping(config, config_in_file)
|
|
815
816
|
|
|
816
817
|
|
|
817
|
-
def override_config_from_env(config:
|
|
818
|
+
def override_config_from_env(config: dict[str, Any]) -> None:
|
|
818
819
|
"""Override a configuration with environment variables values
|
|
819
820
|
|
|
820
821
|
:param config: An eodag providers configuration dictionary
|
|
821
822
|
"""
|
|
822
823
|
|
|
823
824
|
def build_mapping_from_env(
|
|
824
|
-
env_var: str, env_value: str, mapping:
|
|
825
|
+
env_var: str, env_value: str, mapping: dict[str, Any]
|
|
825
826
|
) -> None:
|
|
826
827
|
"""Recursively build a dictionary from an environment variable.
|
|
827
828
|
|
|
@@ -872,7 +873,7 @@ def override_config_from_env(config: Dict[str, Any]) -> None:
|
|
|
872
873
|
new_map = mapping.setdefault(parts[0], {})
|
|
873
874
|
build_mapping_from_env("__".join(parts[1:]), env_value, new_map)
|
|
874
875
|
|
|
875
|
-
mapping_from_env:
|
|
876
|
+
mapping_from_env: dict[str, Any] = {}
|
|
876
877
|
for env_var in os.environ:
|
|
877
878
|
if env_var.startswith("EODAG__"):
|
|
878
879
|
build_mapping_from_env(
|
|
@@ -885,7 +886,7 @@ def override_config_from_env(config: Dict[str, Any]) -> None:
|
|
|
885
886
|
|
|
886
887
|
|
|
887
888
|
def override_config_from_mapping(
|
|
888
|
-
config:
|
|
889
|
+
config: dict[str, Any], mapping: dict[str, Any]
|
|
889
890
|
) -> None:
|
|
890
891
|
"""Override a configuration with the values in a mapping
|
|
891
892
|
|
|
@@ -923,7 +924,7 @@ def override_config_from_mapping(
|
|
|
923
924
|
)
|
|
924
925
|
|
|
925
926
|
# try overriding conf
|
|
926
|
-
old_conf: Optional[
|
|
927
|
+
old_conf: Optional[dict[str, Any]] = config.get(provider)
|
|
927
928
|
if old_conf is not None:
|
|
928
929
|
old_conf.update(new_conf)
|
|
929
930
|
else:
|
|
@@ -944,7 +945,7 @@ def override_config_from_mapping(
|
|
|
944
945
|
logger.debug(tb.format_exc())
|
|
945
946
|
|
|
946
947
|
|
|
947
|
-
def merge_configs(config:
|
|
948
|
+
def merge_configs(config: dict[str, Any], other_config: dict[str, Any]) -> None:
|
|
948
949
|
"""Override a configuration with the values of another configuration
|
|
949
950
|
|
|
950
951
|
:param config: An eodag providers configuration dictionary
|
|
@@ -976,7 +977,7 @@ def merge_configs(config: Dict[str, Any], other_config: Dict[str, Any]) -> None:
|
|
|
976
977
|
config[provider] = new_conf
|
|
977
978
|
|
|
978
979
|
|
|
979
|
-
def load_yml_config(yml_path: str) ->
|
|
980
|
+
def load_yml_config(yml_path: str) -> dict[Any, Any]:
|
|
980
981
|
"""Load a conf dictionary from given yml absolute path
|
|
981
982
|
|
|
982
983
|
:returns: The yml configuration file
|
|
@@ -985,39 +986,35 @@ def load_yml_config(yml_path: str) -> Dict[Any, Any]:
|
|
|
985
986
|
return dict_items_recursive_apply(config.source, string_to_jsonpath)
|
|
986
987
|
|
|
987
988
|
|
|
988
|
-
def load_stac_config() ->
|
|
989
|
+
def load_stac_config() -> dict[str, Any]:
|
|
989
990
|
"""Load the stac configuration into a dictionary
|
|
990
991
|
|
|
991
992
|
:returns: The stac configuration
|
|
992
993
|
"""
|
|
993
|
-
return load_yml_config(
|
|
994
|
-
resource_filename("eodag", os.path.join("resources/", "stac.yml"))
|
|
995
|
-
)
|
|
994
|
+
return load_yml_config(str(res_files("eodag") / "resources" / "stac.yml"))
|
|
996
995
|
|
|
997
996
|
|
|
998
|
-
def load_stac_api_config() ->
|
|
997
|
+
def load_stac_api_config() -> dict[str, Any]:
|
|
999
998
|
"""Load the stac API configuration into a dictionary
|
|
1000
999
|
|
|
1001
1000
|
:returns: The stac API configuration
|
|
1002
1001
|
"""
|
|
1003
|
-
return load_yml_config(
|
|
1004
|
-
resource_filename("eodag", os.path.join("resources/", "stac_api.yml"))
|
|
1005
|
-
)
|
|
1002
|
+
return load_yml_config(str(res_files("eodag") / "resources" / "stac_api.yml"))
|
|
1006
1003
|
|
|
1007
1004
|
|
|
1008
|
-
def load_stac_provider_config() ->
|
|
1005
|
+
def load_stac_provider_config() -> dict[str, Any]:
|
|
1009
1006
|
"""Load the stac provider configuration into a dictionary
|
|
1010
1007
|
|
|
1011
1008
|
:returns: The stac provider configuration
|
|
1012
1009
|
"""
|
|
1013
1010
|
return SimpleYamlProxyConfig(
|
|
1014
|
-
|
|
1011
|
+
str(res_files("eodag") / "resources" / "stac_provider.yml")
|
|
1015
1012
|
).source
|
|
1016
1013
|
|
|
1017
1014
|
|
|
1018
1015
|
def get_ext_product_types_conf(
|
|
1019
1016
|
conf_uri: str = EXT_PRODUCT_TYPES_CONF_URI,
|
|
1020
|
-
) ->
|
|
1017
|
+
) -> dict[str, Any]:
|
|
1021
1018
|
"""Read external product types conf
|
|
1022
1019
|
|
|
1023
1020
|
:param conf_uri: URI to local or remote configuration file
|
eodag/plugins/apis/base.py
CHANGED
eodag/plugins/apis/ecmwf.py
CHANGED
|
@@ -30,11 +30,7 @@ from pydantic.fields import FieldInfo
|
|
|
30
30
|
from eodag.plugins.apis.base import Api
|
|
31
31
|
from eodag.plugins.search import PreparedSearch
|
|
32
32
|
from eodag.plugins.search.base import Search
|
|
33
|
-
from eodag.plugins.search.build_search_result import
|
|
34
|
-
ECMWF_KEYWORDS,
|
|
35
|
-
ECMWFSearch,
|
|
36
|
-
keywords_to_mdt,
|
|
37
|
-
)
|
|
33
|
+
from eodag.plugins.search.build_search_result import ECMWFSearch, ecmwf_mtd
|
|
38
34
|
from eodag.utils import (
|
|
39
35
|
DEFAULT_DOWNLOAD_TIMEOUT,
|
|
40
36
|
DEFAULT_DOWNLOAD_WAIT,
|
|
@@ -48,13 +44,14 @@ from eodag.utils.exceptions import AuthenticationError, DownloadError
|
|
|
48
44
|
from eodag.utils.logging import get_logging_verbose
|
|
49
45
|
|
|
50
46
|
if TYPE_CHECKING:
|
|
51
|
-
from typing import Any,
|
|
47
|
+
from typing import Any, Optional, Union
|
|
52
48
|
|
|
53
49
|
from requests.auth import AuthBase
|
|
54
50
|
|
|
55
51
|
from eodag.api.product import EOProduct
|
|
56
52
|
from eodag.api.search_result import SearchResult
|
|
57
53
|
from eodag.config import PluginConfig
|
|
54
|
+
from eodag.types import S3SessionKwargs
|
|
58
55
|
from eodag.types.download_args import DownloadConf
|
|
59
56
|
from eodag.utils import DownloadedCallback, ProgressCallback, Unpack
|
|
60
57
|
|
|
@@ -83,7 +80,7 @@ class EcmwfApi(Api, ECMWFSearch):
|
|
|
83
80
|
* :attr:`~eodag.config.PluginConfig.type` (``str``) (**mandatory**): EcmwfApi
|
|
84
81
|
* :attr:`~eodag.config.PluginConfig.auth_endpoint` (``str``) (**mandatory**): url of
|
|
85
82
|
the authentication endpoint of the ecmwf api
|
|
86
|
-
* :attr:`~eodag.config.PluginConfig.metadata_mapping` (``
|
|
83
|
+
* :attr:`~eodag.config.PluginConfig.metadata_mapping` (``dict[str, Union[str, list]]``): how
|
|
87
84
|
parameters should be mapped between the provider and eodag; If a string is given, this is
|
|
88
85
|
the mapping parameter returned by provider -> eodag parameter. If a list with 2 elements
|
|
89
86
|
is given, the first one is the mapping eodag parameter -> provider query parameters
|
|
@@ -93,7 +90,7 @@ class EcmwfApi(Api, ECMWFSearch):
|
|
|
93
90
|
def __init__(self, provider: str, config: PluginConfig) -> None:
|
|
94
91
|
# init self.config.metadata_mapping using Search Base plugin
|
|
95
92
|
config.metadata_mapping = {
|
|
96
|
-
**
|
|
93
|
+
**ecmwf_mtd(),
|
|
97
94
|
**config.metadata_mapping,
|
|
98
95
|
}
|
|
99
96
|
Search.__init__(self, provider, config)
|
|
@@ -104,7 +101,7 @@ class EcmwfApi(Api, ECMWFSearch):
|
|
|
104
101
|
self.config.__dict__.setdefault("pagination", {"next_page_query_obj": "{{}}"})
|
|
105
102
|
self.config.__dict__.setdefault("api_endpoint", "")
|
|
106
103
|
|
|
107
|
-
def do_search(self, *args: Any, **kwargs: Any) ->
|
|
104
|
+
def do_search(self, *args: Any, **kwargs: Any) -> list[dict[str, Any]]:
|
|
108
105
|
"""Should perform the actual search request."""
|
|
109
106
|
return [{}]
|
|
110
107
|
|
|
@@ -112,7 +109,7 @@ class EcmwfApi(Api, ECMWFSearch):
|
|
|
112
109
|
self,
|
|
113
110
|
prep: PreparedSearch = PreparedSearch(),
|
|
114
111
|
**kwargs: Any,
|
|
115
|
-
) ->
|
|
112
|
+
) -> tuple[list[EOProduct], Optional[int]]:
|
|
116
113
|
"""Build ready-to-download SearchResult"""
|
|
117
114
|
|
|
118
115
|
# check productType, dates, geometry, use defaults if not specified
|
|
@@ -145,7 +142,7 @@ class EcmwfApi(Api, ECMWFSearch):
|
|
|
145
142
|
|
|
146
143
|
return ECMWFSearch.query(self, prep, **kwargs)
|
|
147
144
|
|
|
148
|
-
def authenticate(self) ->
|
|
145
|
+
def authenticate(self) -> dict[str, Optional[str]]:
|
|
149
146
|
"""Check credentials and returns information needed for auth
|
|
150
147
|
|
|
151
148
|
:returns: {key, url, email} dictionary
|
|
@@ -176,7 +173,7 @@ class EcmwfApi(Api, ECMWFSearch):
|
|
|
176
173
|
def download(
|
|
177
174
|
self,
|
|
178
175
|
product: EOProduct,
|
|
179
|
-
auth: Optional[Union[AuthBase,
|
|
176
|
+
auth: Optional[Union[AuthBase, S3SessionKwargs]] = None,
|
|
180
177
|
progress_callback: Optional[ProgressCallback] = None,
|
|
181
178
|
wait: float = DEFAULT_DOWNLOAD_WAIT,
|
|
182
179
|
timeout: float = DEFAULT_DOWNLOAD_TIMEOUT,
|
|
@@ -265,13 +262,13 @@ class EcmwfApi(Api, ECMWFSearch):
|
|
|
265
262
|
def download_all(
|
|
266
263
|
self,
|
|
267
264
|
products: SearchResult,
|
|
268
|
-
auth: Optional[Union[AuthBase,
|
|
265
|
+
auth: Optional[Union[AuthBase, S3SessionKwargs]] = None,
|
|
269
266
|
downloaded_callback: Optional[DownloadedCallback] = None,
|
|
270
267
|
progress_callback: Optional[ProgressCallback] = None,
|
|
271
268
|
wait: float = DEFAULT_DOWNLOAD_WAIT,
|
|
272
269
|
timeout: float = DEFAULT_DOWNLOAD_TIMEOUT,
|
|
273
270
|
**kwargs: Unpack[DownloadConf],
|
|
274
|
-
) ->
|
|
271
|
+
) -> list[str]:
|
|
275
272
|
"""
|
|
276
273
|
Download all using parent (base plugin) method
|
|
277
274
|
"""
|
|
@@ -291,7 +288,7 @@ class EcmwfApi(Api, ECMWFSearch):
|
|
|
291
288
|
|
|
292
289
|
def discover_queryables(
|
|
293
290
|
self, **kwargs: Any
|
|
294
|
-
) -> Optional[
|
|
291
|
+
) -> Optional[dict[str, Annotated[Any, FieldInfo]]]:
|
|
295
292
|
"""Fetch queryables list from provider using metadata mapping
|
|
296
293
|
|
|
297
294
|
:param kwargs: additional filters for queryables (`productType` and other search
|