eodag 3.0.1__py3-none-any.whl → 3.1.0b2__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- eodag/api/core.py +164 -127
- eodag/api/product/_assets.py +11 -11
- eodag/api/product/_product.py +45 -30
- eodag/api/product/drivers/__init__.py +81 -4
- eodag/api/product/drivers/base.py +65 -4
- eodag/api/product/drivers/generic.py +65 -0
- eodag/api/product/drivers/sentinel1.py +97 -0
- eodag/api/product/drivers/sentinel2.py +95 -0
- eodag/api/product/metadata_mapping.py +101 -85
- eodag/api/search_result.py +13 -23
- eodag/cli.py +26 -5
- eodag/config.py +78 -81
- eodag/plugins/apis/base.py +1 -1
- eodag/plugins/apis/ecmwf.py +46 -22
- eodag/plugins/apis/usgs.py +16 -15
- eodag/plugins/authentication/aws_auth.py +16 -13
- eodag/plugins/authentication/base.py +5 -3
- eodag/plugins/authentication/header.py +3 -3
- eodag/plugins/authentication/keycloak.py +4 -4
- eodag/plugins/authentication/oauth.py +7 -3
- eodag/plugins/authentication/openid_connect.py +16 -16
- eodag/plugins/authentication/sas_auth.py +4 -4
- eodag/plugins/authentication/token.py +41 -10
- eodag/plugins/authentication/token_exchange.py +1 -1
- eodag/plugins/base.py +4 -4
- eodag/plugins/crunch/base.py +4 -4
- eodag/plugins/crunch/filter_date.py +4 -4
- eodag/plugins/crunch/filter_latest_intersect.py +6 -6
- eodag/plugins/crunch/filter_latest_tpl_name.py +7 -7
- eodag/plugins/crunch/filter_overlap.py +4 -4
- eodag/plugins/crunch/filter_property.py +6 -7
- eodag/plugins/download/aws.py +58 -78
- eodag/plugins/download/base.py +38 -56
- eodag/plugins/download/creodias_s3.py +29 -0
- eodag/plugins/download/http.py +173 -183
- eodag/plugins/download/s3rest.py +10 -11
- eodag/plugins/manager.py +10 -20
- eodag/plugins/search/__init__.py +6 -5
- eodag/plugins/search/base.py +87 -44
- eodag/plugins/search/build_search_result.py +1067 -329
- eodag/plugins/search/cop_marine.py +22 -12
- eodag/plugins/search/creodias_s3.py +9 -73
- eodag/plugins/search/csw.py +11 -11
- eodag/plugins/search/data_request_search.py +16 -15
- eodag/plugins/search/qssearch.py +103 -187
- eodag/plugins/search/stac_list_assets.py +85 -0
- eodag/plugins/search/static_stac_search.py +3 -3
- eodag/resources/ext_product_types.json +1 -1
- eodag/resources/product_types.yml +663 -304
- eodag/resources/providers.yml +823 -1749
- eodag/resources/stac_api.yml +2 -2
- eodag/resources/user_conf_template.yml +11 -0
- eodag/rest/cache.py +2 -2
- eodag/rest/config.py +3 -3
- eodag/rest/core.py +112 -82
- eodag/rest/errors.py +5 -5
- eodag/rest/server.py +33 -14
- eodag/rest/stac.py +40 -38
- eodag/rest/types/collections_search.py +3 -3
- eodag/rest/types/eodag_search.py +29 -23
- eodag/rest/types/queryables.py +15 -16
- eodag/rest/types/stac_search.py +15 -25
- eodag/rest/utils/__init__.py +14 -21
- eodag/rest/utils/cql_evaluate.py +6 -6
- eodag/rest/utils/rfc3339.py +2 -2
- eodag/types/__init__.py +75 -28
- eodag/types/bbox.py +2 -2
- eodag/types/download_args.py +3 -3
- eodag/types/queryables.py +183 -72
- eodag/types/search_args.py +4 -4
- eodag/types/whoosh.py +127 -3
- eodag/utils/__init__.py +152 -50
- eodag/utils/exceptions.py +28 -21
- eodag/utils/import_system.py +2 -2
- eodag/utils/repr.py +65 -6
- eodag/utils/requests.py +13 -13
- eodag/utils/rest.py +2 -2
- eodag/utils/s3.py +208 -0
- eodag/utils/stac_reader.py +10 -10
- {eodag-3.0.1.dist-info → eodag-3.1.0b2.dist-info}/METADATA +77 -76
- eodag-3.1.0b2.dist-info/RECORD +113 -0
- {eodag-3.0.1.dist-info → eodag-3.1.0b2.dist-info}/WHEEL +1 -1
- {eodag-3.0.1.dist-info → eodag-3.1.0b2.dist-info}/entry_points.txt +4 -2
- eodag/utils/constraints.py +0 -244
- eodag-3.0.1.dist-info/RECORD +0 -109
- {eodag-3.0.1.dist-info → eodag-3.1.0b2.dist-info}/LICENSE +0 -0
- {eodag-3.0.1.dist-info → eodag-3.1.0b2.dist-info}/top_level.txt +0 -0
eodag/plugins/apis/ecmwf.py
CHANGED
|
@@ -20,16 +20,21 @@ from __future__ import annotations
|
|
|
20
20
|
import logging
|
|
21
21
|
import os
|
|
22
22
|
from datetime import datetime, timezone
|
|
23
|
-
from typing import TYPE_CHECKING
|
|
23
|
+
from typing import TYPE_CHECKING, Annotated
|
|
24
24
|
|
|
25
25
|
import geojson
|
|
26
26
|
from ecmwfapi import ECMWFDataServer, ECMWFService
|
|
27
27
|
from ecmwfapi.api import APIException, Connection, get_apikey_values
|
|
28
|
+
from pydantic.fields import FieldInfo
|
|
28
29
|
|
|
29
30
|
from eodag.plugins.apis.base import Api
|
|
30
31
|
from eodag.plugins.search import PreparedSearch
|
|
31
32
|
from eodag.plugins.search.base import Search
|
|
32
|
-
from eodag.plugins.search.build_search_result import
|
|
33
|
+
from eodag.plugins.search.build_search_result import (
|
|
34
|
+
ECMWF_KEYWORDS,
|
|
35
|
+
ECMWFSearch,
|
|
36
|
+
keywords_to_mdt,
|
|
37
|
+
)
|
|
33
38
|
from eodag.utils import (
|
|
34
39
|
DEFAULT_DOWNLOAD_TIMEOUT,
|
|
35
40
|
DEFAULT_DOWNLOAD_WAIT,
|
|
@@ -43,13 +48,14 @@ from eodag.utils.exceptions import AuthenticationError, DownloadError
|
|
|
43
48
|
from eodag.utils.logging import get_logging_verbose
|
|
44
49
|
|
|
45
50
|
if TYPE_CHECKING:
|
|
46
|
-
from typing import Any,
|
|
51
|
+
from typing import Any, Optional, Union
|
|
47
52
|
|
|
48
53
|
from requests.auth import AuthBase
|
|
49
54
|
|
|
50
55
|
from eodag.api.product import EOProduct
|
|
51
56
|
from eodag.api.search_result import SearchResult
|
|
52
57
|
from eodag.config import PluginConfig
|
|
58
|
+
from eodag.types import S3SessionKwargs
|
|
53
59
|
from eodag.types.download_args import DownloadConf
|
|
54
60
|
from eodag.utils import DownloadedCallback, ProgressCallback, Unpack
|
|
55
61
|
|
|
@@ -58,7 +64,7 @@ logger = logging.getLogger("eodag.apis.ecmwf")
|
|
|
58
64
|
ECMWF_MARS_KNOWN_FORMATS = {"grib": "grib", "netcdf": "nc"}
|
|
59
65
|
|
|
60
66
|
|
|
61
|
-
class EcmwfApi(Api,
|
|
67
|
+
class EcmwfApi(Api, ECMWFSearch):
|
|
62
68
|
"""A plugin that enables to build download-request and download data on ECMWF MARS.
|
|
63
69
|
|
|
64
70
|
Builds a single ready-to-download :class:`~eodag.api.product._product.EOProduct`
|
|
@@ -69,15 +75,16 @@ class EcmwfApi(Api, BuildPostSearchResult):
|
|
|
69
75
|
query).
|
|
70
76
|
|
|
71
77
|
This class inherits from :class:`~eodag.plugins.apis.base.Api` for compatibility and
|
|
72
|
-
:class:`~eodag.plugins.search.build_search_result.
|
|
78
|
+
:class:`~eodag.plugins.search.build_search_result.ECMWFSearch` for the creation
|
|
73
79
|
of the search result.
|
|
74
80
|
|
|
75
81
|
:param provider: provider name
|
|
76
82
|
:param config: Api plugin configuration:
|
|
77
83
|
|
|
78
84
|
* :attr:`~eodag.config.PluginConfig.type` (``str``) (**mandatory**): EcmwfApi
|
|
79
|
-
* :attr:`~eodag.config.PluginConfig.
|
|
80
|
-
|
|
85
|
+
* :attr:`~eodag.config.PluginConfig.auth_endpoint` (``str``) (**mandatory**): url of
|
|
86
|
+
the authentication endpoint of the ecmwf api
|
|
87
|
+
* :attr:`~eodag.config.PluginConfig.metadata_mapping` (``dict[str, Union[str, list]]``): how
|
|
81
88
|
parameters should be mapped between the provider and eodag; If a string is given, this is
|
|
82
89
|
the mapping parameter returned by provider -> eodag parameter. If a list with 2 elements
|
|
83
90
|
is given, the first one is the mapping eodag parameter -> provider query parameters
|
|
@@ -86,14 +93,19 @@ class EcmwfApi(Api, BuildPostSearchResult):
|
|
|
86
93
|
|
|
87
94
|
def __init__(self, provider: str, config: PluginConfig) -> None:
|
|
88
95
|
# init self.config.metadata_mapping using Search Base plugin
|
|
96
|
+
config.metadata_mapping = {
|
|
97
|
+
**keywords_to_mdt(ECMWF_KEYWORDS, "ecmwf"),
|
|
98
|
+
**config.metadata_mapping,
|
|
99
|
+
}
|
|
89
100
|
Search.__init__(self, provider, config)
|
|
90
101
|
|
|
91
102
|
# needed by QueryStringSearch.build_query_string / format_free_text_search
|
|
92
103
|
self.config.__dict__.setdefault("free_text_search_operations", {})
|
|
93
104
|
# needed for compatibility
|
|
94
105
|
self.config.__dict__.setdefault("pagination", {"next_page_query_obj": "{{}}"})
|
|
106
|
+
self.config.__dict__.setdefault("api_endpoint", "")
|
|
95
107
|
|
|
96
|
-
def do_search(self, *args: Any, **kwargs: Any) ->
|
|
108
|
+
def do_search(self, *args: Any, **kwargs: Any) -> list[dict[str, Any]]:
|
|
97
109
|
"""Should perform the actual search request."""
|
|
98
110
|
return [{}]
|
|
99
111
|
|
|
@@ -101,16 +113,16 @@ class EcmwfApi(Api, BuildPostSearchResult):
|
|
|
101
113
|
self,
|
|
102
114
|
prep: PreparedSearch = PreparedSearch(),
|
|
103
115
|
**kwargs: Any,
|
|
104
|
-
) ->
|
|
116
|
+
) -> tuple[list[EOProduct], Optional[int]]:
|
|
105
117
|
"""Build ready-to-download SearchResult"""
|
|
106
118
|
|
|
107
119
|
# check productType, dates, geometry, use defaults if not specified
|
|
108
120
|
# productType
|
|
109
121
|
if not kwargs.get("productType"):
|
|
110
122
|
kwargs["productType"] = "%s_%s_%s" % (
|
|
111
|
-
kwargs.get("dataset", "mars"),
|
|
112
|
-
kwargs.get("type", ""),
|
|
113
|
-
kwargs.get("levtype", ""),
|
|
123
|
+
kwargs.get("ecmwf:dataset", "mars"),
|
|
124
|
+
kwargs.get("ecmwf:type", ""),
|
|
125
|
+
kwargs.get("ecmwf:levtype", ""),
|
|
114
126
|
)
|
|
115
127
|
# start date
|
|
116
128
|
if "startTimeFromAscendingNode" not in kwargs:
|
|
@@ -132,9 +144,9 @@ class EcmwfApi(Api, BuildPostSearchResult):
|
|
|
132
144
|
if "geometry" in kwargs:
|
|
133
145
|
kwargs["geometry"] = get_geometry_from_various(geometry=kwargs["geometry"])
|
|
134
146
|
|
|
135
|
-
return
|
|
147
|
+
return ECMWFSearch.query(self, prep, **kwargs)
|
|
136
148
|
|
|
137
|
-
def authenticate(self) ->
|
|
149
|
+
def authenticate(self) -> dict[str, Optional[str]]:
|
|
138
150
|
"""Check credentials and returns information needed for auth
|
|
139
151
|
|
|
140
152
|
:returns: {key, url, email} dictionary
|
|
@@ -143,7 +155,7 @@ class EcmwfApi(Api, BuildPostSearchResult):
|
|
|
143
155
|
# Get credentials from eodag or using ecmwf conf
|
|
144
156
|
email = getattr(self.config, "credentials", {}).get("username", None)
|
|
145
157
|
key = getattr(self.config, "credentials", {}).get("password", None)
|
|
146
|
-
url = getattr(self.config, "
|
|
158
|
+
url = getattr(self.config, "auth_endpoint", None)
|
|
147
159
|
if not all([email, key, url]):
|
|
148
160
|
key, url, email = get_apikey_values()
|
|
149
161
|
|
|
@@ -165,10 +177,10 @@ class EcmwfApi(Api, BuildPostSearchResult):
|
|
|
165
177
|
def download(
|
|
166
178
|
self,
|
|
167
179
|
product: EOProduct,
|
|
168
|
-
auth: Optional[Union[AuthBase,
|
|
180
|
+
auth: Optional[Union[AuthBase, S3SessionKwargs]] = None,
|
|
169
181
|
progress_callback: Optional[ProgressCallback] = None,
|
|
170
|
-
wait:
|
|
171
|
-
timeout:
|
|
182
|
+
wait: float = DEFAULT_DOWNLOAD_WAIT,
|
|
183
|
+
timeout: float = DEFAULT_DOWNLOAD_TIMEOUT,
|
|
172
184
|
**kwargs: Unpack[DownloadConf],
|
|
173
185
|
) -> Optional[str]:
|
|
174
186
|
"""Download data from ECMWF MARS"""
|
|
@@ -254,13 +266,13 @@ class EcmwfApi(Api, BuildPostSearchResult):
|
|
|
254
266
|
def download_all(
|
|
255
267
|
self,
|
|
256
268
|
products: SearchResult,
|
|
257
|
-
auth: Optional[Union[AuthBase,
|
|
269
|
+
auth: Optional[Union[AuthBase, S3SessionKwargs]] = None,
|
|
258
270
|
downloaded_callback: Optional[DownloadedCallback] = None,
|
|
259
271
|
progress_callback: Optional[ProgressCallback] = None,
|
|
260
|
-
wait:
|
|
261
|
-
timeout:
|
|
272
|
+
wait: float = DEFAULT_DOWNLOAD_WAIT,
|
|
273
|
+
timeout: float = DEFAULT_DOWNLOAD_TIMEOUT,
|
|
262
274
|
**kwargs: Unpack[DownloadConf],
|
|
263
|
-
) ->
|
|
275
|
+
) -> list[str]:
|
|
264
276
|
"""
|
|
265
277
|
Download all using parent (base plugin) method
|
|
266
278
|
"""
|
|
@@ -277,3 +289,15 @@ class EcmwfApi(Api, BuildPostSearchResult):
|
|
|
277
289
|
def clear(self) -> None:
|
|
278
290
|
"""Clear search context"""
|
|
279
291
|
pass
|
|
292
|
+
|
|
293
|
+
def discover_queryables(
|
|
294
|
+
self, **kwargs: Any
|
|
295
|
+
) -> Optional[dict[str, Annotated[Any, FieldInfo]]]:
|
|
296
|
+
"""Fetch queryables list from provider using metadata mapping
|
|
297
|
+
|
|
298
|
+
:param kwargs: additional filters for queryables (`productType` and other search
|
|
299
|
+
arguments)
|
|
300
|
+
:returns: fetched queryable parameters dict
|
|
301
|
+
"""
|
|
302
|
+
product_type = kwargs.get("productType", None)
|
|
303
|
+
return self.queryables_from_metadata_mapping(product_type)
|
eodag/plugins/apis/usgs.py
CHANGED
|
@@ -22,7 +22,7 @@ import os
|
|
|
22
22
|
import shutil
|
|
23
23
|
import tarfile
|
|
24
24
|
import zipfile
|
|
25
|
-
from typing import TYPE_CHECKING, Any,
|
|
25
|
+
from typing import TYPE_CHECKING, Any, Optional, Union, cast
|
|
26
26
|
|
|
27
27
|
import requests
|
|
28
28
|
from jsonpath_ng.ext import parse
|
|
@@ -61,6 +61,7 @@ if TYPE_CHECKING:
|
|
|
61
61
|
|
|
62
62
|
from eodag.api.search_result import SearchResult
|
|
63
63
|
from eodag.config import PluginConfig
|
|
64
|
+
from eodag.types import S3SessionKwargs
|
|
64
65
|
from eodag.types.download_args import DownloadConf
|
|
65
66
|
from eodag.utils import DownloadedCallback, Unpack
|
|
66
67
|
|
|
@@ -86,7 +87,7 @@ class UsgsApi(Api):
|
|
|
86
87
|
file should be extracted; default: ``True``
|
|
87
88
|
* :attr:`~eodag.config.PluginConfig.order_enabled` (``bool``): if the product has to
|
|
88
89
|
be ordered to download it; default: ``False``
|
|
89
|
-
* :attr:`~eodag.config.PluginConfig.metadata_mapping` (``
|
|
90
|
+
* :attr:`~eodag.config.PluginConfig.metadata_mapping` (``dict[str, Union[str, list]]``): how
|
|
90
91
|
parameters should be mapped between the provider and eodag; If a string is given, this is
|
|
91
92
|
the mapping parameter returned by provider -> eodag parameter. If a list with 2 elements
|
|
92
93
|
is given, the first one is the mapping eodag parameter -> provider query parameters
|
|
@@ -99,7 +100,7 @@ class UsgsApi(Api):
|
|
|
99
100
|
# Same method as in base.py, Search.__init__()
|
|
100
101
|
# Prepare the metadata mapping
|
|
101
102
|
# Do a shallow copy, the structure is flat enough for this to be sufficient
|
|
102
|
-
metas:
|
|
103
|
+
metas: dict[str, Any] = DEFAULT_METADATA_MAPPING.copy()
|
|
103
104
|
# Update the defaults with the mapping value. This will add any new key
|
|
104
105
|
# added by the provider mapping that is not in the default metadata.
|
|
105
106
|
metas.update(self.config.metadata_mapping)
|
|
@@ -138,7 +139,7 @@ class UsgsApi(Api):
|
|
|
138
139
|
self,
|
|
139
140
|
prep: PreparedSearch = PreparedSearch(),
|
|
140
141
|
**kwargs: Any,
|
|
141
|
-
) ->
|
|
142
|
+
) -> tuple[list[EOProduct], Optional[int]]:
|
|
142
143
|
"""Search for data on USGS catalogues"""
|
|
143
144
|
page = prep.page if prep.page is not None else DEFAULT_PAGE
|
|
144
145
|
items_per_page = (
|
|
@@ -164,7 +165,7 @@ class UsgsApi(Api):
|
|
|
164
165
|
start_date = kwargs.pop("startTimeFromAscendingNode", None)
|
|
165
166
|
end_date = kwargs.pop("completionTimeFromAscendingNode", None)
|
|
166
167
|
geom = kwargs.pop("geometry", None)
|
|
167
|
-
footprint:
|
|
168
|
+
footprint: dict[str, str] = {}
|
|
168
169
|
if hasattr(geom, "bounds"):
|
|
169
170
|
(
|
|
170
171
|
footprint["lonmin"],
|
|
@@ -175,7 +176,7 @@ class UsgsApi(Api):
|
|
|
175
176
|
else:
|
|
176
177
|
footprint = geom
|
|
177
178
|
|
|
178
|
-
final:
|
|
179
|
+
final: list[EOProduct] = []
|
|
179
180
|
if footprint and len(footprint.keys()) == 4: # a rectangle (or bbox)
|
|
180
181
|
lower_left = {
|
|
181
182
|
"longitude": footprint["lonmin"],
|
|
@@ -295,10 +296,10 @@ class UsgsApi(Api):
|
|
|
295
296
|
def download(
|
|
296
297
|
self,
|
|
297
298
|
product: EOProduct,
|
|
298
|
-
auth: Optional[Union[AuthBase,
|
|
299
|
+
auth: Optional[Union[AuthBase, S3SessionKwargs]] = None,
|
|
299
300
|
progress_callback: Optional[ProgressCallback] = None,
|
|
300
|
-
wait:
|
|
301
|
-
timeout:
|
|
301
|
+
wait: float = DEFAULT_DOWNLOAD_WAIT,
|
|
302
|
+
timeout: float = DEFAULT_DOWNLOAD_TIMEOUT,
|
|
302
303
|
**kwargs: Unpack[DownloadConf],
|
|
303
304
|
) -> Optional[str]:
|
|
304
305
|
"""Download data from USGS catalogues"""
|
|
@@ -340,7 +341,7 @@ class UsgsApi(Api):
|
|
|
340
341
|
product.properties["productId"],
|
|
341
342
|
)
|
|
342
343
|
|
|
343
|
-
req_urls:
|
|
344
|
+
req_urls: list[str] = []
|
|
344
345
|
try:
|
|
345
346
|
if len(download_request_results["data"]["preparingDownloads"]) > 0:
|
|
346
347
|
req_urls.extend(
|
|
@@ -375,7 +376,7 @@ class UsgsApi(Api):
|
|
|
375
376
|
logger.debug(f"Downloading {req_url}")
|
|
376
377
|
ssl_verify = getattr(self.config, "ssl_verify", True)
|
|
377
378
|
|
|
378
|
-
@self.
|
|
379
|
+
@self._order_download_retry(product, wait, timeout)
|
|
379
380
|
def download_request(
|
|
380
381
|
product: EOProduct,
|
|
381
382
|
fs_path: str,
|
|
@@ -464,13 +465,13 @@ class UsgsApi(Api):
|
|
|
464
465
|
def download_all(
|
|
465
466
|
self,
|
|
466
467
|
products: SearchResult,
|
|
467
|
-
auth: Optional[Union[AuthBase,
|
|
468
|
+
auth: Optional[Union[AuthBase, S3SessionKwargs]] = None,
|
|
468
469
|
downloaded_callback: Optional[DownloadedCallback] = None,
|
|
469
470
|
progress_callback: Optional[ProgressCallback] = None,
|
|
470
|
-
wait:
|
|
471
|
-
timeout:
|
|
471
|
+
wait: float = DEFAULT_DOWNLOAD_WAIT,
|
|
472
|
+
timeout: float = DEFAULT_DOWNLOAD_TIMEOUT,
|
|
472
473
|
**kwargs: Unpack[DownloadConf],
|
|
473
|
-
) ->
|
|
474
|
+
) -> list[str]:
|
|
474
475
|
"""
|
|
475
476
|
Download all using parent (base plugin) method
|
|
476
477
|
"""
|
|
@@ -17,9 +17,10 @@
|
|
|
17
17
|
# limitations under the License.
|
|
18
18
|
from __future__ import annotations
|
|
19
19
|
|
|
20
|
-
from typing import TYPE_CHECKING,
|
|
20
|
+
from typing import TYPE_CHECKING, Optional, cast
|
|
21
21
|
|
|
22
22
|
from eodag.plugins.authentication.base import Authentication
|
|
23
|
+
from eodag.types import S3SessionKwargs
|
|
23
24
|
|
|
24
25
|
if TYPE_CHECKING:
|
|
25
26
|
from mypy_boto3_s3.client import S3Client
|
|
@@ -53,12 +54,12 @@ class AwsAuth(Authentication):
|
|
|
53
54
|
|
|
54
55
|
def __init__(self, provider: str, config: PluginConfig) -> None:
|
|
55
56
|
super(AwsAuth, self).__init__(provider, config)
|
|
56
|
-
self.aws_access_key_id = None
|
|
57
|
-
self.aws_secret_access_key = None
|
|
58
|
-
self.aws_session_token = None
|
|
59
|
-
self.profile_name = None
|
|
57
|
+
self.aws_access_key_id: Optional[str] = None
|
|
58
|
+
self.aws_secret_access_key: Optional[str] = None
|
|
59
|
+
self.aws_session_token: Optional[str] = None
|
|
60
|
+
self.profile_name: Optional[str] = None
|
|
60
61
|
|
|
61
|
-
def authenticate(self) ->
|
|
62
|
+
def authenticate(self) -> S3SessionKwargs:
|
|
62
63
|
"""Authenticate
|
|
63
64
|
|
|
64
65
|
:returns: dict containing AWS/boto3 non-empty credentials
|
|
@@ -75,10 +76,12 @@ class AwsAuth(Authentication):
|
|
|
75
76
|
)
|
|
76
77
|
self.profile_name = credentials.get("aws_profile", self.profile_name)
|
|
77
78
|
|
|
78
|
-
|
|
79
|
-
|
|
80
|
-
|
|
81
|
-
|
|
82
|
-
|
|
83
|
-
|
|
84
|
-
|
|
79
|
+
auth_dict = cast(
|
|
80
|
+
S3SessionKwargs,
|
|
81
|
+
{
|
|
82
|
+
k: getattr(self, k)
|
|
83
|
+
for k in S3SessionKwargs.__annotations__
|
|
84
|
+
if getattr(self, k, None)
|
|
85
|
+
},
|
|
86
|
+
)
|
|
87
|
+
return auth_dict
|
|
@@ -17,7 +17,7 @@
|
|
|
17
17
|
# limitations under the License.
|
|
18
18
|
from __future__ import annotations
|
|
19
19
|
|
|
20
|
-
from typing import TYPE_CHECKING,
|
|
20
|
+
from typing import TYPE_CHECKING, Union
|
|
21
21
|
|
|
22
22
|
from eodag.plugins.base import PluginTopic
|
|
23
23
|
from eodag.utils.exceptions import MisconfiguredError
|
|
@@ -25,6 +25,8 @@ from eodag.utils.exceptions import MisconfiguredError
|
|
|
25
25
|
if TYPE_CHECKING:
|
|
26
26
|
from requests.auth import AuthBase
|
|
27
27
|
|
|
28
|
+
from eodag.types import S3SessionKwargs
|
|
29
|
+
|
|
28
30
|
|
|
29
31
|
class Authentication(PluginTopic):
|
|
30
32
|
"""Plugins authentication Base plugin
|
|
@@ -34,11 +36,11 @@ class Authentication(PluginTopic):
|
|
|
34
36
|
|
|
35
37
|
* :attr:`~eodag.config.PluginConfig.matching_url` (``str``): URL pattern to match with search plugin endpoint or
|
|
36
38
|
download link
|
|
37
|
-
* :attr:`~eodag.config.PluginConfig.matching_conf` (``
|
|
39
|
+
* :attr:`~eodag.config.PluginConfig.matching_conf` (``dict[str, Any]``): Part of the search or download plugin
|
|
38
40
|
configuration that needs authentication and helps identifying it
|
|
39
41
|
"""
|
|
40
42
|
|
|
41
|
-
def authenticate(self) -> Union[AuthBase,
|
|
43
|
+
def authenticate(self) -> Union[AuthBase, S3SessionKwargs]:
|
|
42
44
|
"""Authenticate"""
|
|
43
45
|
raise NotImplementedError
|
|
44
46
|
|
|
@@ -17,7 +17,7 @@
|
|
|
17
17
|
# limitations under the License.
|
|
18
18
|
from __future__ import annotations
|
|
19
19
|
|
|
20
|
-
from typing import TYPE_CHECKING
|
|
20
|
+
from typing import TYPE_CHECKING
|
|
21
21
|
|
|
22
22
|
from requests.auth import AuthBase
|
|
23
23
|
|
|
@@ -38,7 +38,7 @@ class HTTPHeaderAuth(Authentication):
|
|
|
38
38
|
:param config: Authentication plugin configuration:
|
|
39
39
|
|
|
40
40
|
* :attr:`~eodag.config.PluginConfig.type` (``str``) (**mandatory**): HTTPHeaderAuth
|
|
41
|
-
* :attr:`~eodag.config.PluginConfig.headers` (``
|
|
41
|
+
* :attr:`~eodag.config.PluginConfig.headers` (``dict[str, str]``): dictionary containing
|
|
42
42
|
all keys/value pairs that should be added to the headers
|
|
43
43
|
|
|
44
44
|
Below an example for the configuration in the providers config file is shown::
|
|
@@ -106,7 +106,7 @@ class HTTPHeaderAuth(Authentication):
|
|
|
106
106
|
class HeaderAuth(AuthBase):
|
|
107
107
|
"""HeaderAuth custom authentication class to be used with requests module"""
|
|
108
108
|
|
|
109
|
-
def __init__(self, authentication_headers:
|
|
109
|
+
def __init__(self, authentication_headers: dict[str, str]) -> None:
|
|
110
110
|
self.auth_headers = authentication_headers
|
|
111
111
|
|
|
112
112
|
def __call__(self, request: PreparedRequest) -> PreparedRequest:
|
|
@@ -18,7 +18,7 @@
|
|
|
18
18
|
from __future__ import annotations
|
|
19
19
|
|
|
20
20
|
import logging
|
|
21
|
-
from typing import TYPE_CHECKING, Any
|
|
21
|
+
from typing import TYPE_CHECKING, Any
|
|
22
22
|
|
|
23
23
|
import requests
|
|
24
24
|
|
|
@@ -56,7 +56,7 @@ class KeycloakOIDCPasswordAuth(OIDCRefreshTokenBase):
|
|
|
56
56
|
token should be added to the query string (``qs``) or to the header (``header``)
|
|
57
57
|
* :attr:`~eodag.config.PluginConfig.token_qs_key` (``str``): (**mandatory if token_provision=qs**)
|
|
58
58
|
key of the param added to the query string
|
|
59
|
-
* :attr:`~eodag.config.PluginConfig.allowed_audiences` (``
|
|
59
|
+
* :attr:`~eodag.config.PluginConfig.allowed_audiences` (``list[str]``) (**mandatory**):
|
|
60
60
|
The allowed audiences that have to be present in the user token.
|
|
61
61
|
* :attr:`~eodag.config.PluginConfig.auth_error_code` (``int``): which error code is
|
|
62
62
|
returned in case of an authentication error
|
|
@@ -130,7 +130,7 @@ class KeycloakOIDCPasswordAuth(OIDCRefreshTokenBase):
|
|
|
130
130
|
key=getattr(self.config, "token_qs_key", None),
|
|
131
131
|
)
|
|
132
132
|
|
|
133
|
-
def _request_new_token(self) ->
|
|
133
|
+
def _request_new_token(self) -> dict[str, Any]:
|
|
134
134
|
logger.debug("fetching new access token")
|
|
135
135
|
req_data = {
|
|
136
136
|
"client_id": self.config.client_id,
|
|
@@ -154,7 +154,7 @@ class KeycloakOIDCPasswordAuth(OIDCRefreshTokenBase):
|
|
|
154
154
|
return self._request_new_token_error(e)
|
|
155
155
|
return response.json()
|
|
156
156
|
|
|
157
|
-
def _get_token_with_refresh_token(self) ->
|
|
157
|
+
def _get_token_with_refresh_token(self) -> dict[str, str]:
|
|
158
158
|
logger.debug("fetching access token with refresh token")
|
|
159
159
|
req_data = {
|
|
160
160
|
"client_id": self.config.client_id,
|
|
@@ -17,12 +17,13 @@
|
|
|
17
17
|
# limitations under the License.
|
|
18
18
|
from __future__ import annotations
|
|
19
19
|
|
|
20
|
-
from typing import TYPE_CHECKING,
|
|
20
|
+
from typing import TYPE_CHECKING, Optional
|
|
21
21
|
|
|
22
22
|
from eodag.plugins.authentication.base import Authentication
|
|
23
23
|
|
|
24
24
|
if TYPE_CHECKING:
|
|
25
25
|
from eodag.config import PluginConfig
|
|
26
|
+
from eodag.types import S3SessionKwargs
|
|
26
27
|
|
|
27
28
|
|
|
28
29
|
class OAuth(Authentication):
|
|
@@ -43,9 +44,12 @@ class OAuth(Authentication):
|
|
|
43
44
|
self.access_key: Optional[str] = None
|
|
44
45
|
self.secret_key: Optional[str] = None
|
|
45
46
|
|
|
46
|
-
def authenticate(self) ->
|
|
47
|
+
def authenticate(self) -> S3SessionKwargs:
|
|
47
48
|
"""Authenticate"""
|
|
48
49
|
self.validate_config_credentials()
|
|
49
50
|
self.access_key = self.config.credentials["aws_access_key_id"]
|
|
50
51
|
self.secret_key = self.config.credentials["aws_secret_access_key"]
|
|
51
|
-
return {
|
|
52
|
+
return {
|
|
53
|
+
"aws_access_key_id": self.access_key,
|
|
54
|
+
"aws_secret_access_key": self.secret_key,
|
|
55
|
+
}
|
|
@@ -22,7 +22,7 @@ import re
|
|
|
22
22
|
import string
|
|
23
23
|
from datetime import datetime, timedelta, timezone
|
|
24
24
|
from random import SystemRandom
|
|
25
|
-
from typing import TYPE_CHECKING, Any,
|
|
25
|
+
from typing import TYPE_CHECKING, Any, Optional
|
|
26
26
|
|
|
27
27
|
import jwt
|
|
28
28
|
import requests
|
|
@@ -68,10 +68,10 @@ class OIDCRefreshTokenBase(Authentication):
|
|
|
68
68
|
self.session = requests.Session()
|
|
69
69
|
|
|
70
70
|
self.access_token = ""
|
|
71
|
-
self.access_token_expiration = datetime.min
|
|
71
|
+
self.access_token_expiration = datetime.min.replace(tzinfo=timezone.utc)
|
|
72
72
|
|
|
73
73
|
self.refresh_token = ""
|
|
74
|
-
self.refresh_token_expiration = datetime.min
|
|
74
|
+
self.refresh_token_expiration = datetime.min.replace(tzinfo=timezone.utc)
|
|
75
75
|
|
|
76
76
|
try:
|
|
77
77
|
response = requests.get(self.config.oidc_config_url)
|
|
@@ -88,7 +88,7 @@ class OIDCRefreshTokenBase(Authentication):
|
|
|
88
88
|
self.authorization_endpoint = auth_config["authorization_endpoint"]
|
|
89
89
|
self.algorithms = auth_config["id_token_signing_alg_values_supported"]
|
|
90
90
|
|
|
91
|
-
def decode_jwt_token(self, token: str) ->
|
|
91
|
+
def decode_jwt_token(self, token: str) -> dict[str, Any]:
|
|
92
92
|
"""Decode JWT token."""
|
|
93
93
|
try:
|
|
94
94
|
key = self.jwks_client.get_signing_key_from_jwt(token).key
|
|
@@ -144,13 +144,13 @@ class OIDCRefreshTokenBase(Authentication):
|
|
|
144
144
|
|
|
145
145
|
return self.access_token
|
|
146
146
|
|
|
147
|
-
def _request_new_token(self) ->
|
|
147
|
+
def _request_new_token(self) -> dict[str, str]:
|
|
148
148
|
"""Fetch the access token with a new authentication"""
|
|
149
149
|
raise NotImplementedError(
|
|
150
150
|
"Incomplete OIDC refresh token retrieval mechanism implementation"
|
|
151
151
|
)
|
|
152
152
|
|
|
153
|
-
def _request_new_token_error(self, e: requests.RequestException) ->
|
|
153
|
+
def _request_new_token_error(self, e: requests.RequestException) -> dict[str, str]:
|
|
154
154
|
"""Handle RequestException raised by `self._request_new_token()`"""
|
|
155
155
|
if self.access_token:
|
|
156
156
|
# try using already retrieved token if authenticate() fails (OTP use-case)
|
|
@@ -186,7 +186,7 @@ class OIDCRefreshTokenBase(Authentication):
|
|
|
186
186
|
)
|
|
187
187
|
)
|
|
188
188
|
|
|
189
|
-
def _get_token_with_refresh_token(self) ->
|
|
189
|
+
def _get_token_with_refresh_token(self) -> dict[str, str]:
|
|
190
190
|
"""Fetch the access token with the refresh token"""
|
|
191
191
|
raise NotImplementedError(
|
|
192
192
|
"Incomplete OIDC refresh token retrieval mechanism implementation"
|
|
@@ -241,21 +241,21 @@ class OIDCAuthorizationCodeFlowAuth(OIDCRefreshTokenBase):
|
|
|
241
241
|
authentication_uri_source=config**) The URL of the authentication backend of the OIDC provider
|
|
242
242
|
* :attr:`~eodag.config.PluginConfig.user_consent_form_xpath` (``str``): The xpath to
|
|
243
243
|
the user consent form. The form is searched in the content of the response to the authorization request
|
|
244
|
-
* :attr:`~eodag.config.PluginConfig.user_consent_form_data` (``
|
|
244
|
+
* :attr:`~eodag.config.PluginConfig.user_consent_form_data` (``dict[str, str]``): The data that
|
|
245
245
|
will be passed with the POST request on the form 'action' URL. The data are given as
|
|
246
246
|
key value pairs, the keys representing the data key and the value being either a 'constant'
|
|
247
247
|
string value, or a string of the form 'xpath(<path-to-a-value-to-be-retrieved>)' and representing a
|
|
248
248
|
value to be retrieved in the user consent form. The xpath must resolve directly to a
|
|
249
249
|
string value, not to an HTML element. Example: ``xpath(//input[@name="sessionDataKeyConsent"]/@value)``
|
|
250
|
-
* :attr:`~eodag.config.PluginConfig.additional_login_form_data` (``
|
|
250
|
+
* :attr:`~eodag.config.PluginConfig.additional_login_form_data` (``dict[str, str]``): A mapping
|
|
251
251
|
giving additional data to be passed to the login POST request. The value follows
|
|
252
252
|
the same rules as with user_consent_form_data
|
|
253
|
-
* :attr:`~eodag.config.PluginConfig.exchange_url_error_pattern` (``
|
|
253
|
+
* :attr:`~eodag.config.PluginConfig.exchange_url_error_pattern` (``dict[str, str]``): Key/value
|
|
254
254
|
pairs of patterns/messages. If exchange_url contains the given pattern, the associated
|
|
255
255
|
message will be sent in an AuthenticationError
|
|
256
256
|
* :attr:`~eodag.config.PluginConfig.client_secret` (``str``): The OIDC provider's client
|
|
257
257
|
secret of the eodag provider
|
|
258
|
-
* :attr:`~eodag.config.PluginConfig.token_exchange_params` (``
|
|
258
|
+
* :attr:`~eodag.config.PluginConfig.token_exchange_params` (``dict[str, str]``): mandatory
|
|
259
259
|
keys for the dict: redirect_uri, client_id; A mapping between OIDC url query string
|
|
260
260
|
and token handler query string params (only necessary if they are not the same as for OIDC).
|
|
261
261
|
This is eodag provider dependant
|
|
@@ -298,7 +298,7 @@ class OIDCAuthorizationCodeFlowAuth(OIDCRefreshTokenBase):
|
|
|
298
298
|
key=getattr(self.config, "token_qs_key", None),
|
|
299
299
|
)
|
|
300
300
|
|
|
301
|
-
def _request_new_token(self) ->
|
|
301
|
+
def _request_new_token(self) -> dict[str, str]:
|
|
302
302
|
"""Fetch the access token with a new authentication"""
|
|
303
303
|
logger.debug("Fetching access token from %s", self.token_endpoint)
|
|
304
304
|
state = self.compute_state()
|
|
@@ -326,12 +326,12 @@ class OIDCAuthorizationCodeFlowAuth(OIDCRefreshTokenBase):
|
|
|
326
326
|
return self._request_new_token_error(e)
|
|
327
327
|
return token_response.json()
|
|
328
328
|
|
|
329
|
-
def _get_token_with_refresh_token(self) ->
|
|
329
|
+
def _get_token_with_refresh_token(self) -> dict[str, str]:
|
|
330
330
|
"""Fetch the access token with the refresh token"""
|
|
331
331
|
logger.debug(
|
|
332
332
|
"Fetching access token with refresh token from %s.", self.token_endpoint
|
|
333
333
|
)
|
|
334
|
-
token_data:
|
|
334
|
+
token_data: dict[str, Any] = {
|
|
335
335
|
"refresh_token": self.refresh_token,
|
|
336
336
|
"grant_type": "refresh_token",
|
|
337
337
|
}
|
|
@@ -435,7 +435,7 @@ class OIDCAuthorizationCodeFlowAuth(OIDCRefreshTokenBase):
|
|
|
435
435
|
verify=ssl_verify,
|
|
436
436
|
)
|
|
437
437
|
|
|
438
|
-
def _prepare_token_post_data(self, token_data:
|
|
438
|
+
def _prepare_token_post_data(self, token_data: dict[str, Any]) -> dict[str, Any]:
|
|
439
439
|
"""Prepare the common data to post to the token URI"""
|
|
440
440
|
token_data.update(
|
|
441
441
|
{
|
|
@@ -471,7 +471,7 @@ class OIDCAuthorizationCodeFlowAuth(OIDCRefreshTokenBase):
|
|
|
471
471
|
"The state received in the authorized url does not match initially computed state"
|
|
472
472
|
)
|
|
473
473
|
code = qs["code"][0]
|
|
474
|
-
token_exchange_data:
|
|
474
|
+
token_exchange_data: dict[str, Any] = {
|
|
475
475
|
"code": code,
|
|
476
476
|
"state": state,
|
|
477
477
|
"grant_type": "authorization_code",
|
|
@@ -19,7 +19,7 @@ from __future__ import annotations
|
|
|
19
19
|
|
|
20
20
|
import logging
|
|
21
21
|
from json import JSONDecodeError
|
|
22
|
-
from typing import TYPE_CHECKING,
|
|
22
|
+
from typing import TYPE_CHECKING, Optional
|
|
23
23
|
|
|
24
24
|
import requests
|
|
25
25
|
from requests.auth import AuthBase
|
|
@@ -42,13 +42,13 @@ class RequestsSASAuth(AuthBase):
|
|
|
42
42
|
self,
|
|
43
43
|
auth_uri: str,
|
|
44
44
|
signed_url_key: str,
|
|
45
|
-
headers: Optional[
|
|
45
|
+
headers: Optional[dict[str, str]] = None,
|
|
46
46
|
ssl_verify: bool = True,
|
|
47
47
|
) -> None:
|
|
48
48
|
self.auth_uri = auth_uri
|
|
49
49
|
self.signed_url_key = signed_url_key
|
|
50
50
|
self.headers = headers
|
|
51
|
-
self.signed_urls:
|
|
51
|
+
self.signed_urls: dict[str, str] = {}
|
|
52
52
|
self.ssl_verify = ssl_verify
|
|
53
53
|
|
|
54
54
|
def __call__(self, request: PreparedRequest) -> PreparedRequest:
|
|
@@ -97,7 +97,7 @@ class SASAuth(Authentication):
|
|
|
97
97
|
get the signed url
|
|
98
98
|
* :attr:`~eodag.config.PluginConfig.signed_url_key` (``str``) (**mandatory**): key to
|
|
99
99
|
get the signed url
|
|
100
|
-
* :attr:`~eodag.config.PluginConfig.headers` (``
|
|
100
|
+
* :attr:`~eodag.config.PluginConfig.headers` (``dict[str, str]``) (**mandatory if
|
|
101
101
|
apiKey is used**): headers to be added to the requests
|
|
102
102
|
* :attr:`~eodag.config.PluginConfig.ssl_verify` (``bool``): if the ssl certificates should be
|
|
103
103
|
verified in the requests; default: ``True``
|