eodag 3.0.0b3__py3-none-any.whl → 3.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- eodag/api/core.py +347 -247
- eodag/api/product/_assets.py +44 -15
- eodag/api/product/_product.py +58 -47
- eodag/api/product/drivers/__init__.py +81 -4
- eodag/api/product/drivers/base.py +65 -4
- eodag/api/product/drivers/generic.py +65 -0
- eodag/api/product/drivers/sentinel1.py +97 -0
- eodag/api/product/drivers/sentinel2.py +95 -0
- eodag/api/product/metadata_mapping.py +129 -93
- eodag/api/search_result.py +28 -12
- eodag/cli.py +61 -24
- eodag/config.py +457 -167
- eodag/plugins/apis/base.py +10 -4
- eodag/plugins/apis/ecmwf.py +53 -23
- eodag/plugins/apis/usgs.py +41 -17
- eodag/plugins/authentication/aws_auth.py +30 -18
- eodag/plugins/authentication/base.py +14 -3
- eodag/plugins/authentication/generic.py +14 -3
- eodag/plugins/authentication/header.py +14 -6
- eodag/plugins/authentication/keycloak.py +44 -25
- eodag/plugins/authentication/oauth.py +18 -4
- eodag/plugins/authentication/openid_connect.py +192 -171
- eodag/plugins/authentication/qsauth.py +12 -4
- eodag/plugins/authentication/sas_auth.py +22 -5
- eodag/plugins/authentication/token.py +95 -17
- eodag/plugins/authentication/token_exchange.py +19 -19
- eodag/plugins/base.py +4 -4
- eodag/plugins/crunch/base.py +8 -5
- eodag/plugins/crunch/filter_date.py +9 -6
- eodag/plugins/crunch/filter_latest_intersect.py +9 -8
- eodag/plugins/crunch/filter_latest_tpl_name.py +8 -8
- eodag/plugins/crunch/filter_overlap.py +9 -11
- eodag/plugins/crunch/filter_property.py +10 -10
- eodag/plugins/download/aws.py +181 -105
- eodag/plugins/download/base.py +49 -67
- eodag/plugins/download/creodias_s3.py +40 -2
- eodag/plugins/download/http.py +247 -223
- eodag/plugins/download/s3rest.py +29 -28
- eodag/plugins/manager.py +176 -41
- eodag/plugins/search/__init__.py +6 -5
- eodag/plugins/search/base.py +123 -60
- eodag/plugins/search/build_search_result.py +1046 -355
- eodag/plugins/search/cop_marine.py +132 -39
- eodag/plugins/search/creodias_s3.py +19 -68
- eodag/plugins/search/csw.py +48 -8
- eodag/plugins/search/data_request_search.py +124 -23
- eodag/plugins/search/qssearch.py +531 -310
- eodag/plugins/search/stac_list_assets.py +85 -0
- eodag/plugins/search/static_stac_search.py +23 -24
- eodag/resources/ext_product_types.json +1 -1
- eodag/resources/product_types.yml +1295 -355
- eodag/resources/providers.yml +1819 -3010
- eodag/resources/stac.yml +3 -163
- eodag/resources/stac_api.yml +2 -2
- eodag/resources/user_conf_template.yml +115 -99
- eodag/rest/cache.py +2 -2
- eodag/rest/config.py +3 -4
- eodag/rest/constants.py +0 -1
- eodag/rest/core.py +157 -117
- eodag/rest/errors.py +181 -0
- eodag/rest/server.py +57 -339
- eodag/rest/stac.py +133 -581
- eodag/rest/types/collections_search.py +3 -3
- eodag/rest/types/eodag_search.py +41 -30
- eodag/rest/types/queryables.py +42 -32
- eodag/rest/types/stac_search.py +15 -16
- eodag/rest/utils/__init__.py +14 -21
- eodag/rest/utils/cql_evaluate.py +6 -6
- eodag/rest/utils/rfc3339.py +2 -2
- eodag/types/__init__.py +153 -32
- eodag/types/bbox.py +2 -2
- eodag/types/download_args.py +4 -4
- eodag/types/queryables.py +183 -73
- eodag/types/search_args.py +6 -6
- eodag/types/whoosh.py +127 -3
- eodag/utils/__init__.py +228 -106
- eodag/utils/exceptions.py +47 -26
- eodag/utils/import_system.py +2 -2
- eodag/utils/logging.py +37 -77
- eodag/utils/repr.py +65 -6
- eodag/utils/requests.py +13 -15
- eodag/utils/rest.py +2 -2
- eodag/utils/s3.py +231 -0
- eodag/utils/stac_reader.py +11 -11
- {eodag-3.0.0b3.dist-info → eodag-3.1.0.dist-info}/METADATA +81 -81
- eodag-3.1.0.dist-info/RECORD +113 -0
- {eodag-3.0.0b3.dist-info → eodag-3.1.0.dist-info}/WHEEL +1 -1
- {eodag-3.0.0b3.dist-info → eodag-3.1.0.dist-info}/entry_points.txt +5 -2
- eodag/resources/constraints/climate-dt.json +0 -13
- eodag/resources/constraints/extremes-dt.json +0 -8
- eodag/utils/constraints.py +0 -244
- eodag-3.0.0b3.dist-info/RECORD +0 -110
- {eodag-3.0.0b3.dist-info → eodag-3.1.0.dist-info}/LICENSE +0 -0
- {eodag-3.0.0b3.dist-info → eodag-3.1.0.dist-info}/top_level.txt +0 -0
eodag/config.py
CHANGED
|
@@ -20,16 +20,15 @@ from __future__ import annotations
|
|
|
20
20
|
import logging
|
|
21
21
|
import os
|
|
22
22
|
import tempfile
|
|
23
|
+
from importlib.resources import files as res_files
|
|
23
24
|
from inspect import isclass
|
|
24
25
|
from typing import (
|
|
26
|
+
Annotated,
|
|
25
27
|
Any,
|
|
26
|
-
Dict,
|
|
27
28
|
ItemsView,
|
|
28
29
|
Iterator,
|
|
29
|
-
List,
|
|
30
30
|
Literal,
|
|
31
31
|
Optional,
|
|
32
|
-
Tuple,
|
|
33
32
|
TypedDict,
|
|
34
33
|
Union,
|
|
35
34
|
ValuesView,
|
|
@@ -43,15 +42,11 @@ import yaml.constructor
|
|
|
43
42
|
import yaml.parser
|
|
44
43
|
from annotated_types import Gt
|
|
45
44
|
from jsonpath_ng import JSONPath
|
|
46
|
-
from pkg_resources import resource_filename
|
|
47
|
-
from requests.auth import AuthBase
|
|
48
|
-
from typing_extensions import Doc
|
|
49
45
|
|
|
50
46
|
from eodag.api.product.metadata_mapping import mtd_cfg_as_conversion_and_querypath
|
|
51
47
|
from eodag.utils import (
|
|
52
48
|
HTTP_REQ_TIMEOUT,
|
|
53
49
|
USER_AGENT,
|
|
54
|
-
Annotated,
|
|
55
50
|
cached_yaml_load,
|
|
56
51
|
cached_yaml_load_all,
|
|
57
52
|
cast_scalar_value,
|
|
@@ -59,6 +54,7 @@ from eodag.utils import (
|
|
|
59
54
|
dict_items_recursive_apply,
|
|
60
55
|
merge_mappings,
|
|
61
56
|
slugify,
|
|
57
|
+
sort_dict,
|
|
62
58
|
string_to_jsonpath,
|
|
63
59
|
update_nested_dict,
|
|
64
60
|
uri_to_path,
|
|
@@ -70,6 +66,8 @@ logger = logging.getLogger("eodag.config")
|
|
|
70
66
|
EXT_PRODUCT_TYPES_CONF_URI = (
|
|
71
67
|
"https://cs-si.github.io/eodag/eodag/resources/ext_product_types.json"
|
|
72
68
|
)
|
|
69
|
+
AUTH_TOPIC_KEYS = ("auth", "search_auth", "download_auth")
|
|
70
|
+
PLUGINS_TOPICS_KEYS = ("api", "search", "download") + AUTH_TOPIC_KEYS
|
|
73
71
|
|
|
74
72
|
|
|
75
73
|
class SimpleYamlProxyConfig:
|
|
@@ -78,7 +76,7 @@ class SimpleYamlProxyConfig:
|
|
|
78
76
|
|
|
79
77
|
def __init__(self, conf_file_path: str) -> None:
|
|
80
78
|
try:
|
|
81
|
-
self.source:
|
|
79
|
+
self.source: dict[str, Any] = cached_yaml_load(conf_file_path)
|
|
82
80
|
except yaml.parser.ParserError as e:
|
|
83
81
|
print("Unable to load user configuration file")
|
|
84
82
|
raise e
|
|
@@ -118,20 +116,24 @@ class ProviderConfig(yaml.YAMLObject):
|
|
|
118
116
|
:param products: (optional) The products types supported by the provider
|
|
119
117
|
:param download: (optional) The configuration of a plugin of type Download
|
|
120
118
|
:param auth: (optional) The configuration of a plugin of type Authentication
|
|
119
|
+
:param search_auth: (optional) The configuration of a plugin of type Authentication for search
|
|
120
|
+
:param download_auth: (optional) The configuration of a plugin of type Authentication for download
|
|
121
121
|
:param kwargs: Additional configuration variables for this provider
|
|
122
122
|
"""
|
|
123
123
|
|
|
124
124
|
name: str
|
|
125
125
|
group: str
|
|
126
126
|
priority: int = 0 # Set default priority to 0
|
|
127
|
-
roles:
|
|
127
|
+
roles: list[str]
|
|
128
128
|
description: str
|
|
129
129
|
url: str
|
|
130
130
|
api: PluginConfig
|
|
131
131
|
search: PluginConfig
|
|
132
|
-
products:
|
|
132
|
+
products: dict[str, Any]
|
|
133
133
|
download: PluginConfig
|
|
134
134
|
auth: PluginConfig
|
|
135
|
+
search_auth: PluginConfig
|
|
136
|
+
download_auth: PluginConfig
|
|
135
137
|
product_types_fetched: bool # set in core.update_product_types_list
|
|
136
138
|
|
|
137
139
|
yaml_loader = yaml.Loader
|
|
@@ -149,10 +151,10 @@ class ProviderConfig(yaml.YAMLObject):
|
|
|
149
151
|
return loader.construct_yaml_object(node, cls)
|
|
150
152
|
|
|
151
153
|
@classmethod
|
|
152
|
-
def from_mapping(cls, mapping:
|
|
154
|
+
def from_mapping(cls, mapping: dict[str, Any]) -> ProviderConfig:
|
|
153
155
|
"""Build a :class:`~eodag.config.ProviderConfig` from a mapping"""
|
|
154
156
|
cls.validate(mapping)
|
|
155
|
-
for key in
|
|
157
|
+
for key in PLUGINS_TOPICS_KEYS:
|
|
156
158
|
if key in mapping:
|
|
157
159
|
mapping[key] = PluginConfig.from_mapping(mapping[key])
|
|
158
160
|
c = cls()
|
|
@@ -160,24 +162,23 @@ class ProviderConfig(yaml.YAMLObject):
|
|
|
160
162
|
return c
|
|
161
163
|
|
|
162
164
|
@staticmethod
|
|
163
|
-
def validate(config_keys: Union[
|
|
165
|
+
def validate(config_keys: Union[tuple[str, ...], dict[str, Any]]) -> None:
|
|
164
166
|
"""Validate a :class:`~eodag.config.ProviderConfig`
|
|
165
167
|
|
|
166
168
|
:param config_keys: The configurations keys to validate
|
|
167
169
|
"""
|
|
168
170
|
if "name" not in config_keys:
|
|
169
171
|
raise ValidationError("Provider config must have name key")
|
|
170
|
-
if not any(k in config_keys for k in
|
|
172
|
+
if not any(k in config_keys for k in PLUGINS_TOPICS_KEYS):
|
|
171
173
|
raise ValidationError("A provider must implement at least one plugin")
|
|
172
|
-
|
|
173
|
-
|
|
174
|
-
):
|
|
174
|
+
non_api_keys = [k for k in PLUGINS_TOPICS_KEYS if k != "api"]
|
|
175
|
+
if "api" in config_keys and any(k in config_keys for k in non_api_keys):
|
|
175
176
|
raise ValidationError(
|
|
176
177
|
"A provider implementing an Api plugin must not implement any other "
|
|
177
178
|
"type of plugin"
|
|
178
179
|
)
|
|
179
180
|
|
|
180
|
-
def update(self, mapping: Optional[
|
|
181
|
+
def update(self, mapping: Optional[dict[str, Any]]) -> None:
|
|
181
182
|
"""Update the configuration parameters with values from `mapping`
|
|
182
183
|
|
|
183
184
|
:param mapping: The mapping from which to override configuration parameters
|
|
@@ -189,12 +190,11 @@ class ProviderConfig(yaml.YAMLObject):
|
|
|
189
190
|
{
|
|
190
191
|
key: value
|
|
191
192
|
for key, value in mapping.items()
|
|
192
|
-
if key not in
|
|
193
|
-
and value is not None
|
|
193
|
+
if key not in PLUGINS_TOPICS_KEYS and value is not None
|
|
194
194
|
},
|
|
195
195
|
)
|
|
196
|
-
for key in
|
|
197
|
-
current_value: Optional[
|
|
196
|
+
for key in PLUGINS_TOPICS_KEYS:
|
|
197
|
+
current_value: Optional[dict[str, Any]] = getattr(self, key, None)
|
|
198
198
|
mapping_value = mapping.get(key, {})
|
|
199
199
|
if current_value is not None:
|
|
200
200
|
current_value.update(mapping_value)
|
|
@@ -203,49 +203,119 @@ class ProviderConfig(yaml.YAMLObject):
|
|
|
203
203
|
|
|
204
204
|
|
|
205
205
|
class PluginConfig(yaml.YAMLObject):
|
|
206
|
-
"""Representation of a plugin config
|
|
206
|
+
"""Representation of a plugin config.
|
|
207
207
|
|
|
208
|
-
|
|
209
|
-
:param metadata_mapping: (optional) The mapping between eodag metadata and
|
|
210
|
-
the plugin specific metadata
|
|
211
|
-
:param free_params: (optional) Additional configuration parameters
|
|
208
|
+
This class variables describe available plugins configuration parameters.
|
|
212
209
|
"""
|
|
213
210
|
|
|
214
211
|
class Pagination(TypedDict):
|
|
215
212
|
"""Search pagination configuration"""
|
|
216
213
|
|
|
214
|
+
#: The maximum number of items per page that the provider can handle
|
|
217
215
|
max_items_per_page: int
|
|
216
|
+
#: Key path for the number of total items in the provider result
|
|
218
217
|
total_items_nb_key_path: Union[str, JSONPath]
|
|
218
|
+
#: Key path for the next page URL
|
|
219
219
|
next_page_url_key_path: Union[str, JSONPath]
|
|
220
|
+
#: Key path for the next page POST request query-object (body)
|
|
220
221
|
next_page_query_obj_key_path: Union[str, JSONPath]
|
|
222
|
+
# TODO: change this typing to bool and adapt code to it
|
|
221
223
|
next_page_merge_key_path: Union[str, JSONPath]
|
|
224
|
+
#: Template to add to :attr:`~eodag.config.PluginConfig.Pagination.next_page_url_tpl` to enable count in
|
|
225
|
+
#: search request
|
|
222
226
|
count_tpl: str
|
|
227
|
+
#: The f-string template for pagination requests.
|
|
223
228
|
next_page_url_tpl: str
|
|
229
|
+
#: The query-object for POST pagination requests.
|
|
224
230
|
next_page_query_obj: str
|
|
231
|
+
#: The endpoint for counting the number of items satisfying a request
|
|
225
232
|
count_endpoint: str
|
|
233
|
+
#: Index of the starting page
|
|
226
234
|
start_page: int
|
|
227
235
|
|
|
228
236
|
class Sort(TypedDict):
|
|
229
237
|
"""Configuration for sort during search"""
|
|
230
238
|
|
|
231
|
-
|
|
239
|
+
#: Default sort settings
|
|
240
|
+
sort_by_default: list[tuple[str, str]]
|
|
241
|
+
#: F-string template to add to :attr:`~eodag.config.PluginConfig.Pagination.next_page_url_tpl` to sort search
|
|
242
|
+
#: results
|
|
232
243
|
sort_by_tpl: str
|
|
233
|
-
|
|
234
|
-
|
|
244
|
+
#: Mapping between eodag and provider query parameters used for sort
|
|
245
|
+
sort_param_mapping: dict[str, str]
|
|
246
|
+
#: Mapping between eodag and provider sort-order parameters
|
|
247
|
+
sort_order_mapping: dict[Literal["ascending", "descending"], str]
|
|
248
|
+
#: Maximum number of allowed sort parameters per request
|
|
235
249
|
max_sort_params: Annotated[int, Gt(0)]
|
|
236
250
|
|
|
237
251
|
class DiscoverMetadata(TypedDict):
|
|
238
|
-
"""Configuration for metadata discovery"""
|
|
252
|
+
"""Configuration for metadata discovery (search result properties)"""
|
|
239
253
|
|
|
254
|
+
#: Whether metadata discovery is enabled or not
|
|
240
255
|
auto_discovery: bool
|
|
256
|
+
#: Metadata regex pattern used for discovery in search result properties
|
|
241
257
|
metadata_pattern: str
|
|
258
|
+
#: Configuration/template that will be used to query for a discovered parameter
|
|
242
259
|
search_param: str
|
|
260
|
+
#: Path to the metadata in search result
|
|
243
261
|
metadata_path: str
|
|
244
262
|
|
|
263
|
+
class DiscoverProductTypes(TypedDict, total=False):
|
|
264
|
+
"""Configuration for product types discovery"""
|
|
265
|
+
|
|
266
|
+
#: URL from which the product types can be fetched
|
|
267
|
+
fetch_url: Optional[str]
|
|
268
|
+
#: HTTP method used to fetch product types
|
|
269
|
+
fetch_method: str
|
|
270
|
+
#: Request body to fetch product types using POST method
|
|
271
|
+
fetch_body: dict[str, Any]
|
|
272
|
+
#: Maximum number of connections for concurrent HTTP requests
|
|
273
|
+
max_connections: int
|
|
274
|
+
#: The f-string template for pagination requests.
|
|
275
|
+
next_page_url_tpl: str
|
|
276
|
+
#: Index of the starting page for pagination requests.
|
|
277
|
+
start_page: int
|
|
278
|
+
#: Type of the provider result
|
|
279
|
+
result_type: str
|
|
280
|
+
#: JsonPath to the list of product types
|
|
281
|
+
results_entry: Union[JSONPath, str]
|
|
282
|
+
#: Mapping for the product type id
|
|
283
|
+
generic_product_type_id: str
|
|
284
|
+
#: Mapping for product type metadata (e.g. ``abstract``, ``licence``) which can be parsed from the provider
|
|
285
|
+
#: result
|
|
286
|
+
generic_product_type_parsable_metadata: dict[str, str]
|
|
287
|
+
#: Mapping for product type properties which can be parsed from the result and are not product type metadata
|
|
288
|
+
generic_product_type_parsable_properties: dict[str, str]
|
|
289
|
+
#: Mapping for product type properties which cannot be parsed from the result and are not product type metadata
|
|
290
|
+
generic_product_type_unparsable_properties: dict[str, str]
|
|
291
|
+
#: URL to fetch data for a single collection
|
|
292
|
+
single_collection_fetch_url: str
|
|
293
|
+
#: Query string to be added to the fetch_url to filter for a collection
|
|
294
|
+
single_collection_fetch_qs: str
|
|
295
|
+
#: Mapping for product type metadata returned by the endpoint given in single_collection_fetch_url
|
|
296
|
+
single_product_type_parsable_metadata: dict[str, str]
|
|
297
|
+
|
|
298
|
+
class DiscoverQueryables(TypedDict, total=False):
|
|
299
|
+
"""Configuration for queryables discovery"""
|
|
300
|
+
|
|
301
|
+
#: URL to fetch the queryables valid for all product types
|
|
302
|
+
fetch_url: Optional[str]
|
|
303
|
+
#: URL to fetch the queryables for a specific product type
|
|
304
|
+
product_type_fetch_url: Optional[str]
|
|
305
|
+
#: Type of the result
|
|
306
|
+
result_type: str
|
|
307
|
+
#: JsonPath to retrieve the queryables from the provider result
|
|
308
|
+
results_entry: str
|
|
309
|
+
#: :class:`~eodag.plugins.search.base.Search` URL of the constraint file used to build queryables
|
|
310
|
+
constraints_url: str
|
|
311
|
+
#: :class:`~eodag.plugins.search.base.Search` Key in the json result where the constraints can be found
|
|
312
|
+
constraints_entry: str
|
|
313
|
+
|
|
245
314
|
class OrderOnResponse(TypedDict):
|
|
246
315
|
"""Configuration for order on-response during download"""
|
|
247
316
|
|
|
248
|
-
|
|
317
|
+
#: Parameters metadata-mapping to apply to the order response
|
|
318
|
+
metadata_mapping: dict[str, Union[str, list[str]]]
|
|
249
319
|
|
|
250
320
|
class OrderStatusSuccess(TypedDict):
|
|
251
321
|
"""
|
|
@@ -255,136 +325,306 @@ class PluginConfig(yaml.YAMLObject):
|
|
|
255
325
|
At least one is required
|
|
256
326
|
"""
|
|
257
327
|
|
|
258
|
-
|
|
259
|
-
|
|
260
|
-
|
|
328
|
+
#: Success value for ``status``
|
|
329
|
+
status: str
|
|
330
|
+
#: Success value for ``message``
|
|
331
|
+
message: str
|
|
332
|
+
#: Success value for status response HTTP code
|
|
333
|
+
http_code: int
|
|
261
334
|
|
|
262
|
-
class OrderStatusOrdered(TypedDict):
|
|
335
|
+
class OrderStatusOrdered(TypedDict, total=False):
|
|
263
336
|
"""
|
|
264
337
|
Configuration to identify order status ordered during download
|
|
265
338
|
"""
|
|
266
339
|
|
|
267
|
-
|
|
340
|
+
#: HTTP code of the order status response
|
|
341
|
+
http_code: int
|
|
268
342
|
|
|
269
|
-
class OrderStatusRequest(TypedDict):
|
|
343
|
+
class OrderStatusRequest(TypedDict, total=False):
|
|
270
344
|
"""
|
|
271
345
|
Order status request configuration
|
|
272
346
|
"""
|
|
273
347
|
|
|
274
|
-
|
|
275
|
-
|
|
348
|
+
#: Request HTTP method
|
|
349
|
+
method: str
|
|
350
|
+
#: Request hearders
|
|
351
|
+
headers: dict[str, Any]
|
|
276
352
|
|
|
277
|
-
class OrderStatusOnSuccess(TypedDict):
|
|
353
|
+
class OrderStatusOnSuccess(TypedDict, total=False):
|
|
278
354
|
"""Configuration for order status on-success during download"""
|
|
279
355
|
|
|
280
|
-
|
|
356
|
+
#: Whether a new search is needed on success or not
|
|
357
|
+
need_search: bool
|
|
358
|
+
#: Return type of the success result
|
|
281
359
|
result_type: str
|
|
360
|
+
#: Key in the success response that gives access to the result
|
|
282
361
|
results_entry: str
|
|
283
|
-
|
|
362
|
+
#: Metadata-mapping to apply to the success status result
|
|
363
|
+
metadata_mapping: dict[str, Union[str, list[str]]]
|
|
284
364
|
|
|
285
|
-
class OrderStatus(TypedDict):
|
|
365
|
+
class OrderStatus(TypedDict, total=False):
|
|
286
366
|
"""Configuration for order status during download"""
|
|
287
367
|
|
|
368
|
+
#: Order status request configuration
|
|
288
369
|
request: PluginConfig.OrderStatusRequest
|
|
289
|
-
|
|
290
|
-
|
|
291
|
-
|
|
292
|
-
]
|
|
370
|
+
#: Metadata-mapping used to parse order status response
|
|
371
|
+
metadata_mapping: dict[str, Union[str, list[str]]]
|
|
372
|
+
#: Configuration to identify order status success during download
|
|
293
373
|
success: PluginConfig.OrderStatusSuccess
|
|
294
|
-
error
|
|
295
|
-
|
|
296
|
-
|
|
297
|
-
]
|
|
374
|
+
#: Part of the order status response that tells there is an error
|
|
375
|
+
error: dict[str, Any]
|
|
376
|
+
#: Configuration to identify order status ordered during download
|
|
298
377
|
ordered: PluginConfig.OrderStatusOrdered
|
|
378
|
+
#: Configuration for order status on-success during download
|
|
299
379
|
on_success: PluginConfig.OrderStatusOnSuccess
|
|
300
380
|
|
|
381
|
+
class MetadataPreMapping(TypedDict, total=False):
|
|
382
|
+
"""Configuration which can be used to simplify further metadata extraction"""
|
|
383
|
+
|
|
384
|
+
#: JsonPath of the metadata entry
|
|
385
|
+
metadata_path: str
|
|
386
|
+
#: Key to get the metadata id
|
|
387
|
+
metadata_path_id: str
|
|
388
|
+
#: Key to get the metadata value
|
|
389
|
+
metadata_path_value: str
|
|
390
|
+
|
|
391
|
+
#: :class:`~eodag.plugins.base.PluginTopic` The name of the plugin class to use to instantiate the plugin object
|
|
301
392
|
name: str
|
|
393
|
+
#: :class:`~eodag.plugins.base.PluginTopic` Plugin type
|
|
302
394
|
type: str
|
|
303
|
-
|
|
304
|
-
|
|
305
|
-
|
|
306
|
-
|
|
307
|
-
|
|
308
|
-
|
|
309
|
-
|
|
310
|
-
|
|
395
|
+
#: :class:`~eodag.plugins.base.PluginTopic` Whether the ssl certificates should be verified in the request or not
|
|
396
|
+
ssl_verify: bool
|
|
397
|
+
#: :class:`~eodag.plugins.base.PluginTopic` Default s3 bucket
|
|
398
|
+
s3_bucket: str
|
|
399
|
+
#: :class:`~eodag.plugins.base.PluginTopic` Authentication error codes
|
|
400
|
+
auth_error_code: Union[int, list[int]]
|
|
401
|
+
#: :class:`~eodag.plugins.base.PluginTopic` Time to wait until request timeout in seconds
|
|
402
|
+
timeout: float
|
|
403
|
+
#: :class:`~eodag.plugins.base.PluginTopic` :class:`urllib3.util.Retry` ``total`` parameter,
|
|
404
|
+
#: total number of retries to allow
|
|
405
|
+
retry_total: int
|
|
406
|
+
#: :class:`~eodag.plugins.base.PluginTopic` :class:`urllib3.util.Retry` ``backoff_factor`` parameter,
|
|
407
|
+
#: backoff factor to apply between attempts after the second try
|
|
408
|
+
retry_backoff_factor: int
|
|
409
|
+
#: :class:`~eodag.plugins.base.PluginTopic` :class:`urllib3.util.Retry` ``status_forcelist`` parameter,
|
|
410
|
+
#: list of integer HTTP status codes that we should force a retry on
|
|
411
|
+
retry_status_forcelist: list[int]
|
|
412
|
+
|
|
413
|
+
# search & api -----------------------------------------------------------------------------------------------------
|
|
414
|
+
# copied from ProviderConfig in PluginManager.get_search_plugins()
|
|
415
|
+
priority: int
|
|
416
|
+
# per product type metadata-mapping, set in core._prepare_search
|
|
417
|
+
product_type_config: dict[str, Any]
|
|
418
|
+
|
|
419
|
+
#: :class:`~eodag.plugins.search.base.Search` Plugin API endpoint
|
|
311
420
|
api_endpoint: str
|
|
421
|
+
#: :class:`~eodag.plugins.search.base.Search` Whether Search plugin needs authentification or not
|
|
312
422
|
need_auth: bool
|
|
423
|
+
#: :class:`~eodag.plugins.search.base.Search` Return type of the provider result
|
|
313
424
|
result_type: str
|
|
425
|
+
#: :class:`~eodag.plugins.search.base.Search`
|
|
426
|
+
#: Key in the provider search result that gives access to the result entries
|
|
314
427
|
results_entry: str
|
|
428
|
+
#: :class:`~eodag.plugins.search.base.Search` Dict containing parameters for pagination
|
|
315
429
|
pagination: PluginConfig.Pagination
|
|
430
|
+
#: :class:`~eodag.plugins.search.base.Search` Configuration for sorting the results
|
|
316
431
|
sort: PluginConfig.Sort
|
|
317
|
-
|
|
432
|
+
#: :class:`~eodag.plugins.search.base.Search` Configuration for the metadata auto-discovery
|
|
318
433
|
discover_metadata: PluginConfig.DiscoverMetadata
|
|
319
|
-
|
|
320
|
-
|
|
321
|
-
|
|
322
|
-
|
|
323
|
-
|
|
324
|
-
|
|
325
|
-
|
|
326
|
-
|
|
327
|
-
|
|
328
|
-
|
|
329
|
-
|
|
330
|
-
|
|
331
|
-
|
|
332
|
-
|
|
333
|
-
|
|
334
|
-
|
|
335
|
-
|
|
336
|
-
|
|
337
|
-
|
|
338
|
-
|
|
339
|
-
|
|
340
|
-
|
|
434
|
+
#: :class:`~eodag.plugins.search.base.Search` Configuration for the product types auto-discovery
|
|
435
|
+
discover_product_types: PluginConfig.DiscoverProductTypes
|
|
436
|
+
#: :class:`~eodag.plugins.search.base.Search` Configuration for the queryables auto-discovery
|
|
437
|
+
discover_queryables: PluginConfig.DiscoverQueryables
|
|
438
|
+
#: :class:`~eodag.plugins.search.base.Search` The mapping between eodag metadata and the plugin specific metadata
|
|
439
|
+
metadata_mapping: dict[str, Union[str, list[str]]]
|
|
440
|
+
#: :class:`~eodag.plugins.search.base.Search` Parameters to remove from queryables
|
|
441
|
+
remove_from_queryables: list[str]
|
|
442
|
+
#: :class:`~eodag.plugins.search.base.Search` Parameters to be passed as is in the search url query string
|
|
443
|
+
literal_search_params: dict[str, str]
|
|
444
|
+
#: :class:`~eodag.plugins.search.qssearch.QueryStringSearch` Characters that should not be quoted in the url params
|
|
445
|
+
dont_quote: list[str]
|
|
446
|
+
#: :class:`~eodag.plugins.search.qssearch.ODataV4Search` Dict describing free text search request build
|
|
447
|
+
free_text_search_operations: dict[str, Any]
|
|
448
|
+
#: :class:`~eodag.plugins.search.qssearch.ODataV4Search` Set to ``True`` if the metadata is not given in the search
|
|
449
|
+
#: result and a two step search has to be performed
|
|
450
|
+
per_product_metadata_query: bool
|
|
451
|
+
#: :class:`~eodag.plugins.search.qssearch.ODataV4Search` Dict used to simplify further metadata extraction
|
|
452
|
+
metadata_pre_mapping: PluginConfig.MetadataPreMapping
|
|
453
|
+
#: :class:`~eodag.plugins.search.data_request_search.DataRequestSearch` URL to which the data request shall be sent
|
|
454
|
+
data_request_url: str
|
|
455
|
+
#: :class:`~eodag.plugins.search.data_request_search.DataRequestSearch` URL to fetch the status of the data request
|
|
456
|
+
status_url: str
|
|
457
|
+
#: :class:`~eodag.plugins.search.data_request_search.DataRequestSearch`
|
|
458
|
+
#: URL to fetch the search result when the data request is done
|
|
459
|
+
result_url: str
|
|
460
|
+
#: :class:`~eodag.plugins.search.data_request_search.DataRequestSearch`
|
|
461
|
+
#: if date parameters are mandatory in the request
|
|
462
|
+
dates_required: bool
|
|
463
|
+
#: :class:`~eodag.plugins.search.csw.CSWSearch` Search definition dictionary
|
|
464
|
+
search_definition: dict[str, Any]
|
|
465
|
+
#: :class:`~eodag.plugins.search.qssearch.PostJsonSearch` Whether to merge responses or not (`aws_eos` specific)
|
|
466
|
+
merge_responses: bool
|
|
467
|
+
#: :class:`~eodag.plugins.search.qssearch.PostJsonSearch` Collections names (`aws_eos` specific)
|
|
468
|
+
collection: list[str]
|
|
469
|
+
#: :class:`~eodag.plugins.search.static_stac_search.StaticStacSearch`
|
|
470
|
+
#: Maximum number of connections for concurrent HTTP requests
|
|
471
|
+
max_connections: int
|
|
472
|
+
#: :class:`~eodag.plugins.search.build_search_result.ECMWFSearch`
|
|
473
|
+
#: Whether end date should be excluded from search request or not
|
|
474
|
+
end_date_excluded: bool
|
|
475
|
+
#: :class:`~eodag.plugins.search.build_search_result.ECMWFSearch`
|
|
476
|
+
#: List of parameters used to parse metadata but that must not be included to the query
|
|
477
|
+
remove_from_query: list[str]
|
|
478
|
+
#: :class:`~eodag.plugins.search.csw.CSWSearch`
|
|
479
|
+
#: OGC Catalogue Service version
|
|
480
|
+
version: str
|
|
481
|
+
#: :class:`~eodag.plugins.apis.ecmwf.EcmwfApi` url of the authentication endpoint
|
|
482
|
+
auth_endpoint: str
|
|
483
|
+
|
|
484
|
+
# download ---------------------------------------------------------------------------------------------------------
|
|
485
|
+
#: :class:`~eodag.plugins.download.base.Download` Default endpoint url
|
|
341
486
|
base_uri: str
|
|
487
|
+
#: :class:`~eodag.plugins.download.base.Download` Where to store downloaded products, as an absolute file path
|
|
342
488
|
output_dir: str
|
|
489
|
+
#: :class:`~eodag.plugins.download.base.Download`
|
|
490
|
+
#: Whether the content of the downloaded file should be extracted or not
|
|
343
491
|
extract: bool
|
|
492
|
+
#: :class:`~eodag.plugins.download.base.Download` Which extension should be used for the downloaded file
|
|
344
493
|
output_extension: str
|
|
345
|
-
|
|
346
|
-
|
|
347
|
-
|
|
348
|
-
|
|
494
|
+
#: :class:`~eodag.plugins.download.base.Download` Whether the directory structure should be flattened or not
|
|
495
|
+
flatten_top_dirs: bool
|
|
496
|
+
#: :class:`~eodag.plugins.download.base.Download` Level in extracted path tree where to find data
|
|
497
|
+
archive_depth: int
|
|
498
|
+
#: :class:`~eodag.plugins.download.base.Download` Whether ignore assets and download using ``downloadLink`` or not
|
|
499
|
+
ignore_assets: bool
|
|
500
|
+
#: :class:`~eodag.plugins.download.base.Download` Product type specific configuration
|
|
501
|
+
products: dict[str, dict[str, Any]]
|
|
502
|
+
#: :class:`~eodag.plugins.download.http.HTTPDownload` Whether the product has to be ordered to download it or not
|
|
503
|
+
order_enabled: bool
|
|
504
|
+
#: :class:`~eodag.plugins.download.http.HTTPDownload` HTTP request method for the order request
|
|
505
|
+
order_method: str
|
|
506
|
+
#: :class:`~eodag.plugins.download.http.HTTPDownload` Headers to be added to the order request
|
|
507
|
+
order_headers: dict[str, str]
|
|
508
|
+
#: :class:`~eodag.plugins.download.http.HTTPDownload`
|
|
509
|
+
#: Dictionary containing the key :attr:`~eodag.config.PluginConfig.metadata_mapping` which can be used to add new
|
|
510
|
+
#: product properties based on the data in response to the order request
|
|
349
511
|
order_on_response: PluginConfig.OrderOnResponse
|
|
512
|
+
#: :class:`~eodag.plugins.download.http.HTTPDownload` Order status handling
|
|
350
513
|
order_status: PluginConfig.OrderStatus
|
|
351
|
-
|
|
352
|
-
|
|
353
|
-
|
|
354
|
-
|
|
355
|
-
|
|
356
|
-
|
|
357
|
-
|
|
358
|
-
|
|
359
|
-
|
|
360
|
-
|
|
361
|
-
|
|
362
|
-
|
|
514
|
+
#: :class:`~eodag.plugins.download.http.HTTPDownload`
|
|
515
|
+
#: Do not authenticate the download request but only the order and order status ones
|
|
516
|
+
no_auth_download: bool
|
|
517
|
+
#: :class:`~eodag.plugins.download.http.HTTPDownload` Parameters to be added to the query params of the request
|
|
518
|
+
dl_url_params: dict[str, str]
|
|
519
|
+
#: :class:`~eodag.plugins.download.s3rest.S3RestDownload`
|
|
520
|
+
#: At which level of the path part of the url the bucket can be found
|
|
521
|
+
bucket_path_level: int
|
|
522
|
+
#: :class:`~eodag.plugins.download.aws.AwsDownload` Whether download is done from a requester-pays bucket or not
|
|
523
|
+
requester_pays: bool
|
|
524
|
+
#: :class:`~eodag.plugins.download.aws.AwsDownload` S3 endpoint
|
|
525
|
+
s3_endpoint: str
|
|
526
|
+
|
|
527
|
+
# auth -------------------------------------------------------------------------------------------------------------
|
|
528
|
+
#: :class:`~eodag.plugins.authentication.base.Authentication` Authentication credentials dictionary
|
|
529
|
+
credentials: dict[str, str]
|
|
530
|
+
#: :class:`~eodag.plugins.authentication.base.Authentication` Authentication URL
|
|
363
531
|
auth_uri: str
|
|
364
|
-
|
|
365
|
-
|
|
366
|
-
headers:
|
|
367
|
-
|
|
368
|
-
|
|
369
|
-
|
|
370
|
-
|
|
371
|
-
|
|
372
|
-
|
|
373
|
-
|
|
374
|
-
|
|
375
|
-
|
|
376
|
-
|
|
377
|
-
|
|
378
|
-
|
|
379
|
-
|
|
380
|
-
|
|
381
|
-
|
|
382
|
-
|
|
383
|
-
|
|
384
|
-
|
|
385
|
-
|
|
386
|
-
|
|
387
|
-
|
|
532
|
+
#: :class:`~eodag.plugins.authentication.base.Authentication`
|
|
533
|
+
#: Dictionary containing all keys/value pairs that should be added to the headers
|
|
534
|
+
headers: dict[str, str]
|
|
535
|
+
#: :class:`~eodag.plugins.authentication.base.Authentication`
|
|
536
|
+
#: Dictionary containing all keys/value pairs that should be added to the headers for token retrieve only
|
|
537
|
+
retrieve_headers: dict[str, str]
|
|
538
|
+
#: :class:`~eodag.plugins.authentication.base.Authentication`
|
|
539
|
+
#: The key pointing to the token in the response from the token server
|
|
540
|
+
token_key: str
|
|
541
|
+
#: :class:`~eodag.plugins.authentication.base.Authentication`
|
|
542
|
+
#: Key to get the refresh token in the response from the token server
|
|
543
|
+
refresh_token_key: str
|
|
544
|
+
#: :class:`~eodag.plugins.authentication.base.Authentication` URL pattern to match with search plugin endpoint or
|
|
545
|
+
#: download link
|
|
546
|
+
matching_url: str
|
|
547
|
+
#: :class:`~eodag.plugins.authentication.base.Authentication` Part of the search or download plugin configuration
|
|
548
|
+
#: that needs authentication
|
|
549
|
+
matching_conf: dict[str, Any]
|
|
550
|
+
#: :class:`~eodag.plugins.authentication.openid_connect.OIDCRefreshTokenBase`
|
|
551
|
+
#: How the token should be used in the request
|
|
552
|
+
token_provision: str
|
|
553
|
+
#: :class:`~eodag.plugins.authentication.openid_connect.OIDCRefreshTokenBase` The OIDC provider's client ID
|
|
554
|
+
client_id: str
|
|
555
|
+
#: :class:`~eodag.plugins.authentication.openid_connect.OIDCRefreshTokenBase` The OIDC provider's client secret
|
|
556
|
+
client_secret: str
|
|
557
|
+
#: :class:`~eodag.plugins.authentication.openid_connect.OIDCRefreshTokenBase`
|
|
558
|
+
#: The OIDC provider's ``.well-known/openid-configuration`` url.
|
|
559
|
+
oidc_config_url: str
|
|
560
|
+
#: :class:`~eodag.plugins.authentication.openid_connect.OIDCRefreshTokenBase` The OIDC token audiences
|
|
561
|
+
allowed_audiences: list[str]
|
|
562
|
+
#: :class:`~eodag.plugins.authentication.openid_connect.OIDCAuthorizationCodeFlowAuth`
|
|
563
|
+
#: Whether a user consent is needed during the authentication or not
|
|
564
|
+
user_consent_needed: str
|
|
565
|
+
#: :class:`~eodag.plugins.authentication.openid_connect.OIDCAuthorizationCodeFlowAuth`
|
|
566
|
+
#: Where to look for the :attr:`~eodag.config.PluginConfig.authorization_uri`
|
|
567
|
+
authentication_uri_source: str
|
|
568
|
+
#: :class:`~eodag.plugins.authentication.openid_connect.OIDCAuthorizationCodeFlowAuth`
|
|
569
|
+
#: The callback url that will handle the code given by the OIDC provider
|
|
570
|
+
authentication_uri: str
|
|
571
|
+
#: :class:`~eodag.plugins.authentication.openid_connect.OIDCAuthorizationCodeFlowAuth`
|
|
572
|
+
#: The URL of the authentication backend of the OIDC provider
|
|
573
|
+
redirect_uri: str
|
|
574
|
+
#: :class:`~eodag.plugins.authentication.openid_connect.OIDCAuthorizationCodeFlowAuth`
|
|
575
|
+
#: The authorization url of the server (where to query for grants)
|
|
576
|
+
authorization_uri: str
|
|
577
|
+
#: :class:`~eodag.plugins.authentication.openid_connect.OIDCAuthorizationCodeFlowAuth`
|
|
578
|
+
#: The xpath to the HTML form element representing the user login form
|
|
579
|
+
login_form_xpath: str
|
|
580
|
+
#: :class:`~eodag.plugins.authentication.openid_connect.OIDCAuthorizationCodeFlowAuth`
|
|
581
|
+
#: The xpath to the user consent form
|
|
582
|
+
user_consent_form_xpath: str
|
|
583
|
+
#: :class:`~eodag.plugins.authentication.openid_connect.OIDCAuthorizationCodeFlowAuth`
|
|
584
|
+
#: The data that will be passed with the POST request on the form 'action' URL
|
|
585
|
+
user_consent_form_data: dict[str, str]
|
|
586
|
+
#: :class:`~eodag.plugins.authentication.openid_connect.OIDCAuthorizationCodeFlowAuth`
|
|
587
|
+
#: Additional data to be passed to the login POST request
|
|
588
|
+
additional_login_form_data: dict[str, str]
|
|
589
|
+
#: :class:`~eodag.plugins.authentication.openid_connect.OIDCAuthorizationCodeFlowAuth`
|
|
590
|
+
#: Key/value pairs of patterns/messages used for Authentication errors
|
|
591
|
+
exchange_url_error_pattern: dict[str, str]
|
|
592
|
+
#: :class:`~eodag.plugins.authentication.openid_connect.OIDCAuthorizationCodeFlowAuth`
|
|
593
|
+
#: A mapping between OIDC url query string and token handler query string params
|
|
594
|
+
token_exchange_params: dict[str, str]
|
|
595
|
+
#: :class:`~eodag.plugins.authentication.openid_connect.OIDCAuthorizationCodeFlowAuth`
|
|
596
|
+
#: Refers to the name of the query param to be used in the query request
|
|
597
|
+
token_qs_key: str
|
|
598
|
+
#: :class:`~eodag.plugins.authentication.openid_connect.OIDCAuthorizationCodeFlowAuth`
|
|
599
|
+
#: Way to pass the data to the POST request that is made to the token server
|
|
600
|
+
token_exchange_post_data_method: str
|
|
601
|
+
#: :class:`~eodag.plugins.authentication.openid_connect.OIDCAuthorizationCodeFlowAuth`
|
|
602
|
+
#: The url to query to get the authorized token
|
|
603
|
+
token_uri: str
|
|
604
|
+
#: :class:`~eodag.plugins.authentication.sas_auth.SASAuth` Key to get the signed url
|
|
605
|
+
signed_url_key: str
|
|
606
|
+
#: :class:`~eodag.plugins.authentication.token.TokenAuth`
|
|
607
|
+
#: Credentials json structure if they should be sent as POST data
|
|
608
|
+
req_data: dict[str, Any]
|
|
609
|
+
#: :class:`~eodag.plugins.authentication.token.TokenAuth`
|
|
610
|
+
#: URL used to fetch the access token with a refresh token
|
|
611
|
+
refresh_uri: str
|
|
612
|
+
#: :class:`~eodag.plugins.authentication.token.TokenAuth`
|
|
613
|
+
#: type of the token
|
|
614
|
+
token_type: str
|
|
615
|
+
#: :class:`~eodag.plugins.authentication.token_exchange.OIDCTokenExchangeAuth`
|
|
616
|
+
#: The full :class:`~eodag.plugins.authentication.openid_connect.OIDCAuthorizationCodeFlowAuth` plugin configuration
|
|
617
|
+
#: used to retrieve subject token
|
|
618
|
+
subject: dict[str, Any]
|
|
619
|
+
#: :class:`~eodag.plugins.authentication.token_exchange.OIDCTokenExchangeAuth`
|
|
620
|
+
#: Identifies the issuer of the `subject_token`
|
|
621
|
+
subject_issuer: str
|
|
622
|
+
#: :class:`~eodag.plugins.authentication.token_exchange.OIDCTokenExchangeAuth`
|
|
623
|
+
#: Audience that the token ID is intended for. :attr:`~eodag.config.PluginConfig.client_id` of the Relying Party
|
|
624
|
+
audience: str
|
|
625
|
+
#: :class:`~eodag.plugins.authentication.generic.GenericAuth`
|
|
626
|
+
#: which authentication method should be used
|
|
627
|
+
method: str
|
|
388
628
|
|
|
389
629
|
yaml_loader = yaml.Loader
|
|
390
630
|
yaml_dumper = yaml.SafeDumper
|
|
@@ -397,21 +637,21 @@ class PluginConfig(yaml.YAMLObject):
|
|
|
397
637
|
return loader.construct_yaml_object(node, cls)
|
|
398
638
|
|
|
399
639
|
@classmethod
|
|
400
|
-
def from_mapping(cls, mapping:
|
|
640
|
+
def from_mapping(cls, mapping: dict[str, Any]) -> PluginConfig:
|
|
401
641
|
"""Build a :class:`~eodag.config.PluginConfig` from a mapping"""
|
|
402
642
|
c = cls()
|
|
403
643
|
c.__dict__.update(mapping)
|
|
404
644
|
return c
|
|
405
645
|
|
|
406
646
|
@staticmethod
|
|
407
|
-
def validate(config_keys:
|
|
647
|
+
def validate(config_keys: tuple[Any, ...]) -> None:
|
|
408
648
|
"""Validate a :class:`~eodag.config.PluginConfig`"""
|
|
409
649
|
if "type" not in config_keys:
|
|
410
650
|
raise ValidationError(
|
|
411
651
|
"A Plugin config must specify the Plugin it configures"
|
|
412
652
|
)
|
|
413
653
|
|
|
414
|
-
def update(self, mapping: Optional[
|
|
654
|
+
def update(self, mapping: Optional[dict[Any, Any]]) -> None:
|
|
415
655
|
"""Update the configuration parameters with values from `mapping`
|
|
416
656
|
|
|
417
657
|
:param mapping: The mapping from which to override configuration parameters
|
|
@@ -423,46 +663,99 @@ class PluginConfig(yaml.YAMLObject):
|
|
|
423
663
|
)
|
|
424
664
|
|
|
425
665
|
|
|
426
|
-
def load_default_config() ->
|
|
427
|
-
"""Load the providers configuration into a
|
|
666
|
+
def load_default_config() -> dict[str, ProviderConfig]:
|
|
667
|
+
"""Load the providers configuration into a dictionary.
|
|
428
668
|
|
|
429
669
|
Load from eodag `resources/providers.yml` or `EODAG_PROVIDERS_CFG_FILE` environment
|
|
430
670
|
variable if exists.
|
|
431
671
|
|
|
432
672
|
:returns: The default provider's configuration
|
|
433
673
|
"""
|
|
434
|
-
eodag_providers_cfg_file = os.getenv(
|
|
435
|
-
"
|
|
436
|
-
)
|
|
674
|
+
eodag_providers_cfg_file = os.getenv("EODAG_PROVIDERS_CFG_FILE") or str(
|
|
675
|
+
res_files("eodag") / "resources" / "providers.yml"
|
|
676
|
+
)
|
|
437
677
|
return load_config(eodag_providers_cfg_file)
|
|
438
678
|
|
|
439
679
|
|
|
440
|
-
def load_config(config_path: str) ->
|
|
441
|
-
"""Load the providers configuration into a
|
|
680
|
+
def load_config(config_path: str) -> dict[str, ProviderConfig]:
|
|
681
|
+
"""Load the providers configuration into a dictionary from a given file
|
|
442
682
|
|
|
443
683
|
:param config_path: The path to the provider config file
|
|
444
684
|
:returns: The default provider's configuration
|
|
445
685
|
"""
|
|
446
686
|
logger.debug("Loading configuration from %s", config_path)
|
|
447
|
-
config:
|
|
687
|
+
config: dict[str, ProviderConfig] = {}
|
|
448
688
|
try:
|
|
449
689
|
# Providers configs are stored in this file as separated yaml documents
|
|
450
690
|
# Load all of it
|
|
451
|
-
providers_configs:
|
|
691
|
+
providers_configs: list[ProviderConfig] = cached_yaml_load_all(config_path)
|
|
452
692
|
except yaml.parser.ParserError as e:
|
|
453
693
|
logger.error("Unable to load configuration")
|
|
454
694
|
raise e
|
|
455
695
|
stac_provider_config = load_stac_provider_config()
|
|
456
696
|
for provider_config in providers_configs:
|
|
457
|
-
# for provider_config in copy.deepcopy(providers_configs):
|
|
458
697
|
provider_config_init(provider_config, stac_provider_config)
|
|
459
698
|
config[provider_config.name] = provider_config
|
|
460
699
|
return config
|
|
461
700
|
|
|
462
701
|
|
|
702
|
+
def credentials_in_auth(auth_conf: PluginConfig) -> bool:
|
|
703
|
+
"""Checks if credentials are set for this Authentication plugin configuration
|
|
704
|
+
|
|
705
|
+
:param auth_conf: Authentication plugin configuration
|
|
706
|
+
:returns: True if credentials are set, else False
|
|
707
|
+
"""
|
|
708
|
+
return any(
|
|
709
|
+
c is not None for c in (getattr(auth_conf, "credentials", {}) or {}).values()
|
|
710
|
+
)
|
|
711
|
+
|
|
712
|
+
|
|
713
|
+
def share_credentials(
|
|
714
|
+
providers_config: dict[str, ProviderConfig],
|
|
715
|
+
) -> None:
|
|
716
|
+
"""Share credentials between plugins having the same matching criteria
|
|
717
|
+
|
|
718
|
+
:param providers_configs: eodag providers configurations
|
|
719
|
+
"""
|
|
720
|
+
auth_confs_with_creds = [
|
|
721
|
+
getattr(p, k)
|
|
722
|
+
for p in providers_config.values()
|
|
723
|
+
for k in AUTH_TOPIC_KEYS
|
|
724
|
+
if hasattr(p, k) and credentials_in_auth(getattr(p, k))
|
|
725
|
+
]
|
|
726
|
+
for provider, provider_config in providers_config.items():
|
|
727
|
+
if auth_confs_with_creds:
|
|
728
|
+
for auth_topic_key in AUTH_TOPIC_KEYS:
|
|
729
|
+
provider_config_auth = getattr(provider_config, auth_topic_key, None)
|
|
730
|
+
if provider_config_auth and not credentials_in_auth(
|
|
731
|
+
provider_config_auth
|
|
732
|
+
):
|
|
733
|
+
# no credentials set for this provider
|
|
734
|
+
provider_matching_conf = getattr(
|
|
735
|
+
provider_config_auth, "matching_conf", {}
|
|
736
|
+
)
|
|
737
|
+
provider_matching_url = getattr(
|
|
738
|
+
provider_config_auth, "matching_url", None
|
|
739
|
+
)
|
|
740
|
+
for conf_with_creds in auth_confs_with_creds:
|
|
741
|
+
# copy credentials between plugins if `matching_conf` or `matching_url` are matching
|
|
742
|
+
if (
|
|
743
|
+
provider_matching_conf
|
|
744
|
+
and sort_dict(provider_matching_conf)
|
|
745
|
+
== sort_dict(getattr(conf_with_creds, "matching_conf", {}))
|
|
746
|
+
) or (
|
|
747
|
+
provider_matching_url
|
|
748
|
+
and provider_matching_url
|
|
749
|
+
== getattr(conf_with_creds, "matching_url", None)
|
|
750
|
+
):
|
|
751
|
+
getattr(
|
|
752
|
+
providers_config[provider], auth_topic_key
|
|
753
|
+
).credentials = conf_with_creds.credentials
|
|
754
|
+
|
|
755
|
+
|
|
463
756
|
def provider_config_init(
|
|
464
757
|
provider_config: ProviderConfig,
|
|
465
|
-
stac_search_default_conf: Optional[
|
|
758
|
+
stac_search_default_conf: Optional[dict[str, Any]] = None,
|
|
466
759
|
) -> None:
|
|
467
760
|
"""Applies some default values to provider config
|
|
468
761
|
|
|
@@ -471,13 +764,13 @@ def provider_config_init(
|
|
|
471
764
|
"""
|
|
472
765
|
# For the provider, set the default output_dir of its download plugin
|
|
473
766
|
# as tempdir in a portable way
|
|
474
|
-
for
|
|
475
|
-
if
|
|
476
|
-
|
|
477
|
-
if not getattr(
|
|
478
|
-
|
|
479
|
-
if not getattr(
|
|
480
|
-
|
|
767
|
+
for download_topic_key in ("download", "api"):
|
|
768
|
+
if download_topic_key in vars(provider_config):
|
|
769
|
+
download_conf = getattr(provider_config, download_topic_key)
|
|
770
|
+
if not getattr(download_conf, "output_dir", None):
|
|
771
|
+
download_conf.output_dir = tempfile.gettempdir()
|
|
772
|
+
if not getattr(download_conf, "delete_archive", None):
|
|
773
|
+
download_conf.delete_archive = True
|
|
481
774
|
|
|
482
775
|
try:
|
|
483
776
|
if (
|
|
@@ -486,6 +779,7 @@ def provider_config_init(
|
|
|
486
779
|
and provider_config.search.type
|
|
487
780
|
in [
|
|
488
781
|
"StacSearch",
|
|
782
|
+
"StacListAssets",
|
|
489
783
|
"StaticStacSearch",
|
|
490
784
|
]
|
|
491
785
|
):
|
|
@@ -500,7 +794,7 @@ def provider_config_init(
|
|
|
500
794
|
pass
|
|
501
795
|
|
|
502
796
|
|
|
503
|
-
def override_config_from_file(config:
|
|
797
|
+
def override_config_from_file(config: dict[str, Any], file_path: str) -> None:
|
|
504
798
|
"""Override a configuration with the values in a file
|
|
505
799
|
|
|
506
800
|
:param config: An eodag providers configuration dictionary
|
|
@@ -518,14 +812,14 @@ def override_config_from_file(config: Dict[str, Any], file_path: str) -> None:
|
|
|
518
812
|
override_config_from_mapping(config, config_in_file)
|
|
519
813
|
|
|
520
814
|
|
|
521
|
-
def override_config_from_env(config:
|
|
815
|
+
def override_config_from_env(config: dict[str, Any]) -> None:
|
|
522
816
|
"""Override a configuration with environment variables values
|
|
523
817
|
|
|
524
818
|
:param config: An eodag providers configuration dictionary
|
|
525
819
|
"""
|
|
526
820
|
|
|
527
821
|
def build_mapping_from_env(
|
|
528
|
-
env_var: str, env_value: str, mapping:
|
|
822
|
+
env_var: str, env_value: str, mapping: dict[str, Any]
|
|
529
823
|
) -> None:
|
|
530
824
|
"""Recursively build a dictionary from an environment variable.
|
|
531
825
|
|
|
@@ -576,7 +870,7 @@ def override_config_from_env(config: Dict[str, Any]) -> None:
|
|
|
576
870
|
new_map = mapping.setdefault(parts[0], {})
|
|
577
871
|
build_mapping_from_env("__".join(parts[1:]), env_value, new_map)
|
|
578
872
|
|
|
579
|
-
mapping_from_env:
|
|
873
|
+
mapping_from_env: dict[str, Any] = {}
|
|
580
874
|
for env_var in os.environ:
|
|
581
875
|
if env_var.startswith("EODAG__"):
|
|
582
876
|
build_mapping_from_env(
|
|
@@ -589,7 +883,7 @@ def override_config_from_env(config: Dict[str, Any]) -> None:
|
|
|
589
883
|
|
|
590
884
|
|
|
591
885
|
def override_config_from_mapping(
|
|
592
|
-
config:
|
|
886
|
+
config: dict[str, Any], mapping: dict[str, Any]
|
|
593
887
|
) -> None:
|
|
594
888
|
"""Override a configuration with the values in a mapping
|
|
595
889
|
|
|
@@ -627,7 +921,7 @@ def override_config_from_mapping(
|
|
|
627
921
|
)
|
|
628
922
|
|
|
629
923
|
# try overriding conf
|
|
630
|
-
old_conf: Optional[
|
|
924
|
+
old_conf: Optional[dict[str, Any]] = config.get(provider)
|
|
631
925
|
if old_conf is not None:
|
|
632
926
|
old_conf.update(new_conf)
|
|
633
927
|
else:
|
|
@@ -648,7 +942,7 @@ def override_config_from_mapping(
|
|
|
648
942
|
logger.debug(tb.format_exc())
|
|
649
943
|
|
|
650
944
|
|
|
651
|
-
def merge_configs(config:
|
|
945
|
+
def merge_configs(config: dict[str, Any], other_config: dict[str, Any]) -> None:
|
|
652
946
|
"""Override a configuration with the values of another configuration
|
|
653
947
|
|
|
654
948
|
:param config: An eodag providers configuration dictionary
|
|
@@ -680,8 +974,8 @@ def merge_configs(config: Dict[str, Any], other_config: Dict[str, Any]) -> None:
|
|
|
680
974
|
config[provider] = new_conf
|
|
681
975
|
|
|
682
976
|
|
|
683
|
-
def load_yml_config(yml_path: str) ->
|
|
684
|
-
"""Load a conf
|
|
977
|
+
def load_yml_config(yml_path: str) -> dict[Any, Any]:
|
|
978
|
+
"""Load a conf dictionary from given yml absolute path
|
|
685
979
|
|
|
686
980
|
:returns: The yml configuration file
|
|
687
981
|
"""
|
|
@@ -689,39 +983,35 @@ def load_yml_config(yml_path: str) -> Dict[Any, Any]:
|
|
|
689
983
|
return dict_items_recursive_apply(config.source, string_to_jsonpath)
|
|
690
984
|
|
|
691
985
|
|
|
692
|
-
def load_stac_config() ->
|
|
693
|
-
"""Load the stac configuration into a
|
|
986
|
+
def load_stac_config() -> dict[str, Any]:
|
|
987
|
+
"""Load the stac configuration into a dictionary
|
|
694
988
|
|
|
695
989
|
:returns: The stac configuration
|
|
696
990
|
"""
|
|
697
|
-
return load_yml_config(
|
|
698
|
-
resource_filename("eodag", os.path.join("resources/", "stac.yml"))
|
|
699
|
-
)
|
|
991
|
+
return load_yml_config(str(res_files("eodag") / "resources" / "stac.yml"))
|
|
700
992
|
|
|
701
993
|
|
|
702
|
-
def load_stac_api_config() ->
|
|
703
|
-
"""Load the stac API configuration into a
|
|
994
|
+
def load_stac_api_config() -> dict[str, Any]:
|
|
995
|
+
"""Load the stac API configuration into a dictionary
|
|
704
996
|
|
|
705
997
|
:returns: The stac API configuration
|
|
706
998
|
"""
|
|
707
|
-
return load_yml_config(
|
|
708
|
-
resource_filename("eodag", os.path.join("resources/", "stac_api.yml"))
|
|
709
|
-
)
|
|
999
|
+
return load_yml_config(str(res_files("eodag") / "resources" / "stac_api.yml"))
|
|
710
1000
|
|
|
711
1001
|
|
|
712
|
-
def load_stac_provider_config() ->
|
|
713
|
-
"""Load the stac provider configuration into a
|
|
1002
|
+
def load_stac_provider_config() -> dict[str, Any]:
|
|
1003
|
+
"""Load the stac provider configuration into a dictionary
|
|
714
1004
|
|
|
715
1005
|
:returns: The stac provider configuration
|
|
716
1006
|
"""
|
|
717
1007
|
return SimpleYamlProxyConfig(
|
|
718
|
-
|
|
1008
|
+
str(res_files("eodag") / "resources" / "stac_provider.yml")
|
|
719
1009
|
).source
|
|
720
1010
|
|
|
721
1011
|
|
|
722
1012
|
def get_ext_product_types_conf(
|
|
723
1013
|
conf_uri: str = EXT_PRODUCT_TYPES_CONF_URI,
|
|
724
|
-
) ->
|
|
1014
|
+
) -> dict[str, Any]:
|
|
725
1015
|
"""Read external product types conf
|
|
726
1016
|
|
|
727
1017
|
:param conf_uri: URI to local or remote configuration file
|