eodag 3.1.0b1__py3-none-any.whl → 3.2.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- eodag/api/core.py +69 -63
- eodag/api/product/_assets.py +49 -13
- eodag/api/product/_product.py +41 -30
- eodag/api/product/drivers/__init__.py +81 -4
- eodag/api/product/drivers/base.py +65 -4
- eodag/api/product/drivers/generic.py +65 -0
- eodag/api/product/drivers/sentinel1.py +97 -0
- eodag/api/product/drivers/sentinel2.py +95 -0
- eodag/api/product/metadata_mapping.py +85 -79
- eodag/api/search_result.py +13 -23
- eodag/cli.py +4 -4
- eodag/config.py +77 -80
- eodag/plugins/apis/base.py +1 -1
- eodag/plugins/apis/ecmwf.py +12 -15
- eodag/plugins/apis/usgs.py +12 -11
- eodag/plugins/authentication/aws_auth.py +16 -13
- eodag/plugins/authentication/base.py +5 -3
- eodag/plugins/authentication/header.py +3 -3
- eodag/plugins/authentication/keycloak.py +4 -4
- eodag/plugins/authentication/oauth.py +7 -3
- eodag/plugins/authentication/openid_connect.py +20 -14
- eodag/plugins/authentication/sas_auth.py +4 -4
- eodag/plugins/authentication/token.py +7 -7
- eodag/plugins/authentication/token_exchange.py +1 -1
- eodag/plugins/base.py +4 -4
- eodag/plugins/crunch/base.py +4 -4
- eodag/plugins/crunch/filter_date.py +4 -4
- eodag/plugins/crunch/filter_latest_intersect.py +6 -6
- eodag/plugins/crunch/filter_latest_tpl_name.py +7 -7
- eodag/plugins/crunch/filter_overlap.py +4 -4
- eodag/plugins/crunch/filter_property.py +4 -4
- eodag/plugins/download/aws.py +137 -77
- eodag/plugins/download/base.py +8 -17
- eodag/plugins/download/creodias_s3.py +2 -2
- eodag/plugins/download/http.py +30 -32
- eodag/plugins/download/s3rest.py +5 -4
- eodag/plugins/manager.py +10 -20
- eodag/plugins/search/__init__.py +6 -5
- eodag/plugins/search/base.py +38 -42
- eodag/plugins/search/build_search_result.py +286 -336
- eodag/plugins/search/cop_marine.py +22 -12
- eodag/plugins/search/creodias_s3.py +8 -78
- eodag/plugins/search/csw.py +11 -11
- eodag/plugins/search/data_request_search.py +19 -18
- eodag/plugins/search/qssearch.py +84 -151
- eodag/plugins/search/stac_list_assets.py +85 -0
- eodag/plugins/search/static_stac_search.py +4 -4
- eodag/resources/ext_product_types.json +1 -1
- eodag/resources/product_types.yml +848 -398
- eodag/resources/providers.yml +1038 -1115
- eodag/resources/stac_api.yml +2 -2
- eodag/resources/user_conf_template.yml +10 -9
- eodag/rest/cache.py +2 -2
- eodag/rest/config.py +3 -3
- eodag/rest/core.py +24 -24
- eodag/rest/errors.py +5 -5
- eodag/rest/server.py +3 -11
- eodag/rest/stac.py +41 -38
- eodag/rest/types/collections_search.py +3 -3
- eodag/rest/types/eodag_search.py +23 -23
- eodag/rest/types/queryables.py +40 -28
- eodag/rest/types/stac_search.py +15 -25
- eodag/rest/utils/__init__.py +11 -21
- eodag/rest/utils/cql_evaluate.py +6 -6
- eodag/rest/utils/rfc3339.py +2 -2
- eodag/types/__init__.py +97 -29
- eodag/types/bbox.py +2 -2
- eodag/types/download_args.py +2 -2
- eodag/types/queryables.py +5 -2
- eodag/types/search_args.py +4 -4
- eodag/types/whoosh.py +1 -3
- eodag/utils/__init__.py +82 -41
- eodag/utils/exceptions.py +2 -2
- eodag/utils/import_system.py +2 -2
- eodag/utils/requests.py +2 -2
- eodag/utils/rest.py +2 -2
- eodag/utils/s3.py +231 -0
- eodag/utils/stac_reader.py +10 -10
- {eodag-3.1.0b1.dist-info → eodag-3.2.0.dist-info}/METADATA +12 -10
- eodag-3.2.0.dist-info/RECORD +113 -0
- {eodag-3.1.0b1.dist-info → eodag-3.2.0.dist-info}/WHEEL +1 -1
- {eodag-3.1.0b1.dist-info → eodag-3.2.0.dist-info}/entry_points.txt +1 -0
- eodag-3.1.0b1.dist-info/RECORD +0 -108
- {eodag-3.1.0b1.dist-info → eodag-3.2.0.dist-info/licenses}/LICENSE +0 -0
- {eodag-3.1.0b1.dist-info → eodag-3.2.0.dist-info}/top_level.txt +0 -0
eodag/plugins/apis/usgs.py
CHANGED
|
@@ -22,7 +22,7 @@ import os
|
|
|
22
22
|
import shutil
|
|
23
23
|
import tarfile
|
|
24
24
|
import zipfile
|
|
25
|
-
from typing import TYPE_CHECKING, Any,
|
|
25
|
+
from typing import TYPE_CHECKING, Any, Optional, Union, cast
|
|
26
26
|
|
|
27
27
|
import requests
|
|
28
28
|
from jsonpath_ng.ext import parse
|
|
@@ -61,6 +61,7 @@ if TYPE_CHECKING:
|
|
|
61
61
|
|
|
62
62
|
from eodag.api.search_result import SearchResult
|
|
63
63
|
from eodag.config import PluginConfig
|
|
64
|
+
from eodag.types import S3SessionKwargs
|
|
64
65
|
from eodag.types.download_args import DownloadConf
|
|
65
66
|
from eodag.utils import DownloadedCallback, Unpack
|
|
66
67
|
|
|
@@ -86,7 +87,7 @@ class UsgsApi(Api):
|
|
|
86
87
|
file should be extracted; default: ``True``
|
|
87
88
|
* :attr:`~eodag.config.PluginConfig.order_enabled` (``bool``): if the product has to
|
|
88
89
|
be ordered to download it; default: ``False``
|
|
89
|
-
* :attr:`~eodag.config.PluginConfig.metadata_mapping` (``
|
|
90
|
+
* :attr:`~eodag.config.PluginConfig.metadata_mapping` (``dict[str, Union[str, list]]``): how
|
|
90
91
|
parameters should be mapped between the provider and eodag; If a string is given, this is
|
|
91
92
|
the mapping parameter returned by provider -> eodag parameter. If a list with 2 elements
|
|
92
93
|
is given, the first one is the mapping eodag parameter -> provider query parameters
|
|
@@ -99,7 +100,7 @@ class UsgsApi(Api):
|
|
|
99
100
|
# Same method as in base.py, Search.__init__()
|
|
100
101
|
# Prepare the metadata mapping
|
|
101
102
|
# Do a shallow copy, the structure is flat enough for this to be sufficient
|
|
102
|
-
metas:
|
|
103
|
+
metas: dict[str, Any] = DEFAULT_METADATA_MAPPING.copy()
|
|
103
104
|
# Update the defaults with the mapping value. This will add any new key
|
|
104
105
|
# added by the provider mapping that is not in the default metadata.
|
|
105
106
|
metas.update(self.config.metadata_mapping)
|
|
@@ -127,7 +128,7 @@ class UsgsApi(Api):
|
|
|
127
128
|
api.logout()
|
|
128
129
|
continue
|
|
129
130
|
except USGSError as e:
|
|
130
|
-
if i == 0:
|
|
131
|
+
if i == 0 and os.path.isfile(api.TMPFILE):
|
|
131
132
|
# `.usgs` API file key might be obsolete
|
|
132
133
|
# Remove it and try again
|
|
133
134
|
os.remove(api.TMPFILE)
|
|
@@ -138,7 +139,7 @@ class UsgsApi(Api):
|
|
|
138
139
|
self,
|
|
139
140
|
prep: PreparedSearch = PreparedSearch(),
|
|
140
141
|
**kwargs: Any,
|
|
141
|
-
) ->
|
|
142
|
+
) -> tuple[list[EOProduct], Optional[int]]:
|
|
142
143
|
"""Search for data on USGS catalogues"""
|
|
143
144
|
page = prep.page if prep.page is not None else DEFAULT_PAGE
|
|
144
145
|
items_per_page = (
|
|
@@ -164,7 +165,7 @@ class UsgsApi(Api):
|
|
|
164
165
|
start_date = kwargs.pop("startTimeFromAscendingNode", None)
|
|
165
166
|
end_date = kwargs.pop("completionTimeFromAscendingNode", None)
|
|
166
167
|
geom = kwargs.pop("geometry", None)
|
|
167
|
-
footprint:
|
|
168
|
+
footprint: dict[str, str] = {}
|
|
168
169
|
if hasattr(geom, "bounds"):
|
|
169
170
|
(
|
|
170
171
|
footprint["lonmin"],
|
|
@@ -175,7 +176,7 @@ class UsgsApi(Api):
|
|
|
175
176
|
else:
|
|
176
177
|
footprint = geom
|
|
177
178
|
|
|
178
|
-
final:
|
|
179
|
+
final: list[EOProduct] = []
|
|
179
180
|
if footprint and len(footprint.keys()) == 4: # a rectangle (or bbox)
|
|
180
181
|
lower_left = {
|
|
181
182
|
"longitude": footprint["lonmin"],
|
|
@@ -295,7 +296,7 @@ class UsgsApi(Api):
|
|
|
295
296
|
def download(
|
|
296
297
|
self,
|
|
297
298
|
product: EOProduct,
|
|
298
|
-
auth: Optional[Union[AuthBase,
|
|
299
|
+
auth: Optional[Union[AuthBase, S3SessionKwargs]] = None,
|
|
299
300
|
progress_callback: Optional[ProgressCallback] = None,
|
|
300
301
|
wait: float = DEFAULT_DOWNLOAD_WAIT,
|
|
301
302
|
timeout: float = DEFAULT_DOWNLOAD_TIMEOUT,
|
|
@@ -340,7 +341,7 @@ class UsgsApi(Api):
|
|
|
340
341
|
product.properties["productId"],
|
|
341
342
|
)
|
|
342
343
|
|
|
343
|
-
req_urls:
|
|
344
|
+
req_urls: list[str] = []
|
|
344
345
|
try:
|
|
345
346
|
if len(download_request_results["data"]["preparingDownloads"]) > 0:
|
|
346
347
|
req_urls.extend(
|
|
@@ -464,13 +465,13 @@ class UsgsApi(Api):
|
|
|
464
465
|
def download_all(
|
|
465
466
|
self,
|
|
466
467
|
products: SearchResult,
|
|
467
|
-
auth: Optional[Union[AuthBase,
|
|
468
|
+
auth: Optional[Union[AuthBase, S3SessionKwargs]] = None,
|
|
468
469
|
downloaded_callback: Optional[DownloadedCallback] = None,
|
|
469
470
|
progress_callback: Optional[ProgressCallback] = None,
|
|
470
471
|
wait: float = DEFAULT_DOWNLOAD_WAIT,
|
|
471
472
|
timeout: float = DEFAULT_DOWNLOAD_TIMEOUT,
|
|
472
473
|
**kwargs: Unpack[DownloadConf],
|
|
473
|
-
) ->
|
|
474
|
+
) -> list[str]:
|
|
474
475
|
"""
|
|
475
476
|
Download all using parent (base plugin) method
|
|
476
477
|
"""
|
|
@@ -17,9 +17,10 @@
|
|
|
17
17
|
# limitations under the License.
|
|
18
18
|
from __future__ import annotations
|
|
19
19
|
|
|
20
|
-
from typing import TYPE_CHECKING,
|
|
20
|
+
from typing import TYPE_CHECKING, Optional, cast
|
|
21
21
|
|
|
22
22
|
from eodag.plugins.authentication.base import Authentication
|
|
23
|
+
from eodag.types import S3SessionKwargs
|
|
23
24
|
|
|
24
25
|
if TYPE_CHECKING:
|
|
25
26
|
from mypy_boto3_s3.client import S3Client
|
|
@@ -53,12 +54,12 @@ class AwsAuth(Authentication):
|
|
|
53
54
|
|
|
54
55
|
def __init__(self, provider: str, config: PluginConfig) -> None:
|
|
55
56
|
super(AwsAuth, self).__init__(provider, config)
|
|
56
|
-
self.aws_access_key_id = None
|
|
57
|
-
self.aws_secret_access_key = None
|
|
58
|
-
self.aws_session_token = None
|
|
59
|
-
self.profile_name = None
|
|
57
|
+
self.aws_access_key_id: Optional[str] = None
|
|
58
|
+
self.aws_secret_access_key: Optional[str] = None
|
|
59
|
+
self.aws_session_token: Optional[str] = None
|
|
60
|
+
self.profile_name: Optional[str] = None
|
|
60
61
|
|
|
61
|
-
def authenticate(self) ->
|
|
62
|
+
def authenticate(self) -> S3SessionKwargs:
|
|
62
63
|
"""Authenticate
|
|
63
64
|
|
|
64
65
|
:returns: dict containing AWS/boto3 non-empty credentials
|
|
@@ -75,10 +76,12 @@ class AwsAuth(Authentication):
|
|
|
75
76
|
)
|
|
76
77
|
self.profile_name = credentials.get("aws_profile", self.profile_name)
|
|
77
78
|
|
|
78
|
-
|
|
79
|
-
|
|
80
|
-
|
|
81
|
-
|
|
82
|
-
|
|
83
|
-
|
|
84
|
-
|
|
79
|
+
auth_dict = cast(
|
|
80
|
+
S3SessionKwargs,
|
|
81
|
+
{
|
|
82
|
+
k: getattr(self, k)
|
|
83
|
+
for k in S3SessionKwargs.__annotations__
|
|
84
|
+
if getattr(self, k, None)
|
|
85
|
+
},
|
|
86
|
+
)
|
|
87
|
+
return auth_dict
|
|
@@ -17,7 +17,7 @@
|
|
|
17
17
|
# limitations under the License.
|
|
18
18
|
from __future__ import annotations
|
|
19
19
|
|
|
20
|
-
from typing import TYPE_CHECKING,
|
|
20
|
+
from typing import TYPE_CHECKING, Union
|
|
21
21
|
|
|
22
22
|
from eodag.plugins.base import PluginTopic
|
|
23
23
|
from eodag.utils.exceptions import MisconfiguredError
|
|
@@ -25,6 +25,8 @@ from eodag.utils.exceptions import MisconfiguredError
|
|
|
25
25
|
if TYPE_CHECKING:
|
|
26
26
|
from requests.auth import AuthBase
|
|
27
27
|
|
|
28
|
+
from eodag.types import S3SessionKwargs
|
|
29
|
+
|
|
28
30
|
|
|
29
31
|
class Authentication(PluginTopic):
|
|
30
32
|
"""Plugins authentication Base plugin
|
|
@@ -34,11 +36,11 @@ class Authentication(PluginTopic):
|
|
|
34
36
|
|
|
35
37
|
* :attr:`~eodag.config.PluginConfig.matching_url` (``str``): URL pattern to match with search plugin endpoint or
|
|
36
38
|
download link
|
|
37
|
-
* :attr:`~eodag.config.PluginConfig.matching_conf` (``
|
|
39
|
+
* :attr:`~eodag.config.PluginConfig.matching_conf` (``dict[str, Any]``): Part of the search or download plugin
|
|
38
40
|
configuration that needs authentication and helps identifying it
|
|
39
41
|
"""
|
|
40
42
|
|
|
41
|
-
def authenticate(self) -> Union[AuthBase,
|
|
43
|
+
def authenticate(self) -> Union[AuthBase, S3SessionKwargs]:
|
|
42
44
|
"""Authenticate"""
|
|
43
45
|
raise NotImplementedError
|
|
44
46
|
|
|
@@ -17,7 +17,7 @@
|
|
|
17
17
|
# limitations under the License.
|
|
18
18
|
from __future__ import annotations
|
|
19
19
|
|
|
20
|
-
from typing import TYPE_CHECKING
|
|
20
|
+
from typing import TYPE_CHECKING
|
|
21
21
|
|
|
22
22
|
from requests.auth import AuthBase
|
|
23
23
|
|
|
@@ -38,7 +38,7 @@ class HTTPHeaderAuth(Authentication):
|
|
|
38
38
|
:param config: Authentication plugin configuration:
|
|
39
39
|
|
|
40
40
|
* :attr:`~eodag.config.PluginConfig.type` (``str``) (**mandatory**): HTTPHeaderAuth
|
|
41
|
-
* :attr:`~eodag.config.PluginConfig.headers` (``
|
|
41
|
+
* :attr:`~eodag.config.PluginConfig.headers` (``dict[str, str]``): dictionary containing
|
|
42
42
|
all keys/value pairs that should be added to the headers
|
|
43
43
|
|
|
44
44
|
Below an example for the configuration in the providers config file is shown::
|
|
@@ -106,7 +106,7 @@ class HTTPHeaderAuth(Authentication):
|
|
|
106
106
|
class HeaderAuth(AuthBase):
|
|
107
107
|
"""HeaderAuth custom authentication class to be used with requests module"""
|
|
108
108
|
|
|
109
|
-
def __init__(self, authentication_headers:
|
|
109
|
+
def __init__(self, authentication_headers: dict[str, str]) -> None:
|
|
110
110
|
self.auth_headers = authentication_headers
|
|
111
111
|
|
|
112
112
|
def __call__(self, request: PreparedRequest) -> PreparedRequest:
|
|
@@ -18,7 +18,7 @@
|
|
|
18
18
|
from __future__ import annotations
|
|
19
19
|
|
|
20
20
|
import logging
|
|
21
|
-
from typing import TYPE_CHECKING, Any
|
|
21
|
+
from typing import TYPE_CHECKING, Any
|
|
22
22
|
|
|
23
23
|
import requests
|
|
24
24
|
|
|
@@ -56,7 +56,7 @@ class KeycloakOIDCPasswordAuth(OIDCRefreshTokenBase):
|
|
|
56
56
|
token should be added to the query string (``qs``) or to the header (``header``)
|
|
57
57
|
* :attr:`~eodag.config.PluginConfig.token_qs_key` (``str``): (**mandatory if token_provision=qs**)
|
|
58
58
|
key of the param added to the query string
|
|
59
|
-
* :attr:`~eodag.config.PluginConfig.allowed_audiences` (``
|
|
59
|
+
* :attr:`~eodag.config.PluginConfig.allowed_audiences` (``list[str]``) (**mandatory**):
|
|
60
60
|
The allowed audiences that have to be present in the user token.
|
|
61
61
|
* :attr:`~eodag.config.PluginConfig.auth_error_code` (``int``): which error code is
|
|
62
62
|
returned in case of an authentication error
|
|
@@ -130,7 +130,7 @@ class KeycloakOIDCPasswordAuth(OIDCRefreshTokenBase):
|
|
|
130
130
|
key=getattr(self.config, "token_qs_key", None),
|
|
131
131
|
)
|
|
132
132
|
|
|
133
|
-
def _request_new_token(self) ->
|
|
133
|
+
def _request_new_token(self) -> dict[str, Any]:
|
|
134
134
|
logger.debug("fetching new access token")
|
|
135
135
|
req_data = {
|
|
136
136
|
"client_id": self.config.client_id,
|
|
@@ -154,7 +154,7 @@ class KeycloakOIDCPasswordAuth(OIDCRefreshTokenBase):
|
|
|
154
154
|
return self._request_new_token_error(e)
|
|
155
155
|
return response.json()
|
|
156
156
|
|
|
157
|
-
def _get_token_with_refresh_token(self) ->
|
|
157
|
+
def _get_token_with_refresh_token(self) -> dict[str, str]:
|
|
158
158
|
logger.debug("fetching access token with refresh token")
|
|
159
159
|
req_data = {
|
|
160
160
|
"client_id": self.config.client_id,
|
|
@@ -17,12 +17,13 @@
|
|
|
17
17
|
# limitations under the License.
|
|
18
18
|
from __future__ import annotations
|
|
19
19
|
|
|
20
|
-
from typing import TYPE_CHECKING,
|
|
20
|
+
from typing import TYPE_CHECKING, Optional
|
|
21
21
|
|
|
22
22
|
from eodag.plugins.authentication.base import Authentication
|
|
23
23
|
|
|
24
24
|
if TYPE_CHECKING:
|
|
25
25
|
from eodag.config import PluginConfig
|
|
26
|
+
from eodag.types import S3SessionKwargs
|
|
26
27
|
|
|
27
28
|
|
|
28
29
|
class OAuth(Authentication):
|
|
@@ -43,9 +44,12 @@ class OAuth(Authentication):
|
|
|
43
44
|
self.access_key: Optional[str] = None
|
|
44
45
|
self.secret_key: Optional[str] = None
|
|
45
46
|
|
|
46
|
-
def authenticate(self) ->
|
|
47
|
+
def authenticate(self) -> S3SessionKwargs:
|
|
47
48
|
"""Authenticate"""
|
|
48
49
|
self.validate_config_credentials()
|
|
49
50
|
self.access_key = self.config.credentials["aws_access_key_id"]
|
|
50
51
|
self.secret_key = self.config.credentials["aws_secret_access_key"]
|
|
51
|
-
return {
|
|
52
|
+
return {
|
|
53
|
+
"aws_access_key_id": self.access_key,
|
|
54
|
+
"aws_secret_access_key": self.secret_key,
|
|
55
|
+
}
|
|
@@ -22,7 +22,7 @@ import re
|
|
|
22
22
|
import string
|
|
23
23
|
from datetime import datetime, timedelta, timezone
|
|
24
24
|
from random import SystemRandom
|
|
25
|
-
from typing import TYPE_CHECKING, Any,
|
|
25
|
+
from typing import TYPE_CHECKING, Any, Optional
|
|
26
26
|
|
|
27
27
|
import jwt
|
|
28
28
|
import requests
|
|
@@ -88,7 +88,7 @@ class OIDCRefreshTokenBase(Authentication):
|
|
|
88
88
|
self.authorization_endpoint = auth_config["authorization_endpoint"]
|
|
89
89
|
self.algorithms = auth_config["id_token_signing_alg_values_supported"]
|
|
90
90
|
|
|
91
|
-
def decode_jwt_token(self, token: str) ->
|
|
91
|
+
def decode_jwt_token(self, token: str) -> dict[str, Any]:
|
|
92
92
|
"""Decode JWT token."""
|
|
93
93
|
try:
|
|
94
94
|
key = self.jwks_client.get_signing_key_from_jwt(token).key
|
|
@@ -144,13 +144,13 @@ class OIDCRefreshTokenBase(Authentication):
|
|
|
144
144
|
|
|
145
145
|
return self.access_token
|
|
146
146
|
|
|
147
|
-
def _request_new_token(self) ->
|
|
147
|
+
def _request_new_token(self) -> dict[str, str]:
|
|
148
148
|
"""Fetch the access token with a new authentication"""
|
|
149
149
|
raise NotImplementedError(
|
|
150
150
|
"Incomplete OIDC refresh token retrieval mechanism implementation"
|
|
151
151
|
)
|
|
152
152
|
|
|
153
|
-
def _request_new_token_error(self, e: requests.RequestException) ->
|
|
153
|
+
def _request_new_token_error(self, e: requests.RequestException) -> dict[str, str]:
|
|
154
154
|
"""Handle RequestException raised by `self._request_new_token()`"""
|
|
155
155
|
if self.access_token:
|
|
156
156
|
# try using already retrieved token if authenticate() fails (OTP use-case)
|
|
@@ -186,7 +186,7 @@ class OIDCRefreshTokenBase(Authentication):
|
|
|
186
186
|
)
|
|
187
187
|
)
|
|
188
188
|
|
|
189
|
-
def _get_token_with_refresh_token(self) ->
|
|
189
|
+
def _get_token_with_refresh_token(self) -> dict[str, str]:
|
|
190
190
|
"""Fetch the access token with the refresh token"""
|
|
191
191
|
raise NotImplementedError(
|
|
192
192
|
"Incomplete OIDC refresh token retrieval mechanism implementation"
|
|
@@ -241,21 +241,21 @@ class OIDCAuthorizationCodeFlowAuth(OIDCRefreshTokenBase):
|
|
|
241
241
|
authentication_uri_source=config**) The URL of the authentication backend of the OIDC provider
|
|
242
242
|
* :attr:`~eodag.config.PluginConfig.user_consent_form_xpath` (``str``): The xpath to
|
|
243
243
|
the user consent form. The form is searched in the content of the response to the authorization request
|
|
244
|
-
* :attr:`~eodag.config.PluginConfig.user_consent_form_data` (``
|
|
244
|
+
* :attr:`~eodag.config.PluginConfig.user_consent_form_data` (``dict[str, str]``): The data that
|
|
245
245
|
will be passed with the POST request on the form 'action' URL. The data are given as
|
|
246
246
|
key value pairs, the keys representing the data key and the value being either a 'constant'
|
|
247
247
|
string value, or a string of the form 'xpath(<path-to-a-value-to-be-retrieved>)' and representing a
|
|
248
248
|
value to be retrieved in the user consent form. The xpath must resolve directly to a
|
|
249
249
|
string value, not to an HTML element. Example: ``xpath(//input[@name="sessionDataKeyConsent"]/@value)``
|
|
250
|
-
* :attr:`~eodag.config.PluginConfig.additional_login_form_data` (``
|
|
250
|
+
* :attr:`~eodag.config.PluginConfig.additional_login_form_data` (``dict[str, str]``): A mapping
|
|
251
251
|
giving additional data to be passed to the login POST request. The value follows
|
|
252
252
|
the same rules as with user_consent_form_data
|
|
253
|
-
* :attr:`~eodag.config.PluginConfig.exchange_url_error_pattern` (``
|
|
253
|
+
* :attr:`~eodag.config.PluginConfig.exchange_url_error_pattern` (``dict[str, str]``): Key/value
|
|
254
254
|
pairs of patterns/messages. If exchange_url contains the given pattern, the associated
|
|
255
255
|
message will be sent in an AuthenticationError
|
|
256
256
|
* :attr:`~eodag.config.PluginConfig.client_secret` (``str``): The OIDC provider's client
|
|
257
257
|
secret of the eodag provider
|
|
258
|
-
* :attr:`~eodag.config.PluginConfig.token_exchange_params` (``
|
|
258
|
+
* :attr:`~eodag.config.PluginConfig.token_exchange_params` (``dict[str, str]``): mandatory
|
|
259
259
|
keys for the dict: redirect_uri, client_id; A mapping between OIDC url query string
|
|
260
260
|
and token handler query string params (only necessary if they are not the same as for OIDC).
|
|
261
261
|
This is eodag provider dependant
|
|
@@ -298,7 +298,7 @@ class OIDCAuthorizationCodeFlowAuth(OIDCRefreshTokenBase):
|
|
|
298
298
|
key=getattr(self.config, "token_qs_key", None),
|
|
299
299
|
)
|
|
300
300
|
|
|
301
|
-
def _request_new_token(self) ->
|
|
301
|
+
def _request_new_token(self) -> dict[str, str]:
|
|
302
302
|
"""Fetch the access token with a new authentication"""
|
|
303
303
|
logger.debug("Fetching access token from %s", self.token_endpoint)
|
|
304
304
|
state = self.compute_state()
|
|
@@ -326,12 +326,12 @@ class OIDCAuthorizationCodeFlowAuth(OIDCRefreshTokenBase):
|
|
|
326
326
|
return self._request_new_token_error(e)
|
|
327
327
|
return token_response.json()
|
|
328
328
|
|
|
329
|
-
def _get_token_with_refresh_token(self) ->
|
|
329
|
+
def _get_token_with_refresh_token(self) -> dict[str, str]:
|
|
330
330
|
"""Fetch the access token with the refresh token"""
|
|
331
331
|
logger.debug(
|
|
332
332
|
"Fetching access token with refresh token from %s.", self.token_endpoint
|
|
333
333
|
)
|
|
334
|
-
token_data:
|
|
334
|
+
token_data: dict[str, Any] = {
|
|
335
335
|
"refresh_token": self.refresh_token,
|
|
336
336
|
"grant_type": "refresh_token",
|
|
337
337
|
}
|
|
@@ -379,6 +379,12 @@ class OIDCAuthorizationCodeFlowAuth(OIDCRefreshTokenBase):
|
|
|
379
379
|
|
|
380
380
|
login_document = etree.HTML(authorization_response.text)
|
|
381
381
|
login_forms = login_document.xpath(self.config.login_form_xpath)
|
|
382
|
+
|
|
383
|
+
if not login_forms:
|
|
384
|
+
# we assume user is already logged in
|
|
385
|
+
# no form found because we got redirected to the redirect_uri
|
|
386
|
+
return authorization_response
|
|
387
|
+
|
|
382
388
|
login_form = login_forms[0]
|
|
383
389
|
|
|
384
390
|
# Get the form data to pass to the login form from config or from the login form
|
|
@@ -435,7 +441,7 @@ class OIDCAuthorizationCodeFlowAuth(OIDCRefreshTokenBase):
|
|
|
435
441
|
verify=ssl_verify,
|
|
436
442
|
)
|
|
437
443
|
|
|
438
|
-
def _prepare_token_post_data(self, token_data:
|
|
444
|
+
def _prepare_token_post_data(self, token_data: dict[str, Any]) -> dict[str, Any]:
|
|
439
445
|
"""Prepare the common data to post to the token URI"""
|
|
440
446
|
token_data.update(
|
|
441
447
|
{
|
|
@@ -471,7 +477,7 @@ class OIDCAuthorizationCodeFlowAuth(OIDCRefreshTokenBase):
|
|
|
471
477
|
"The state received in the authorized url does not match initially computed state"
|
|
472
478
|
)
|
|
473
479
|
code = qs["code"][0]
|
|
474
|
-
token_exchange_data:
|
|
480
|
+
token_exchange_data: dict[str, Any] = {
|
|
475
481
|
"code": code,
|
|
476
482
|
"state": state,
|
|
477
483
|
"grant_type": "authorization_code",
|
|
@@ -19,7 +19,7 @@ from __future__ import annotations
|
|
|
19
19
|
|
|
20
20
|
import logging
|
|
21
21
|
from json import JSONDecodeError
|
|
22
|
-
from typing import TYPE_CHECKING,
|
|
22
|
+
from typing import TYPE_CHECKING, Optional
|
|
23
23
|
|
|
24
24
|
import requests
|
|
25
25
|
from requests.auth import AuthBase
|
|
@@ -42,13 +42,13 @@ class RequestsSASAuth(AuthBase):
|
|
|
42
42
|
self,
|
|
43
43
|
auth_uri: str,
|
|
44
44
|
signed_url_key: str,
|
|
45
|
-
headers: Optional[
|
|
45
|
+
headers: Optional[dict[str, str]] = None,
|
|
46
46
|
ssl_verify: bool = True,
|
|
47
47
|
) -> None:
|
|
48
48
|
self.auth_uri = auth_uri
|
|
49
49
|
self.signed_url_key = signed_url_key
|
|
50
50
|
self.headers = headers
|
|
51
|
-
self.signed_urls:
|
|
51
|
+
self.signed_urls: dict[str, str] = {}
|
|
52
52
|
self.ssl_verify = ssl_verify
|
|
53
53
|
|
|
54
54
|
def __call__(self, request: PreparedRequest) -> PreparedRequest:
|
|
@@ -97,7 +97,7 @@ class SASAuth(Authentication):
|
|
|
97
97
|
get the signed url
|
|
98
98
|
* :attr:`~eodag.config.PluginConfig.signed_url_key` (``str``) (**mandatory**): key to
|
|
99
99
|
get the signed url
|
|
100
|
-
* :attr:`~eodag.config.PluginConfig.headers` (``
|
|
100
|
+
* :attr:`~eodag.config.PluginConfig.headers` (``dict[str, str]``) (**mandatory if
|
|
101
101
|
apiKey is used**): headers to be added to the requests
|
|
102
102
|
* :attr:`~eodag.config.PluginConfig.ssl_verify` (``bool``): if the ssl certificates should be
|
|
103
103
|
verified in the requests; default: ``True``
|
|
@@ -18,7 +18,7 @@
|
|
|
18
18
|
from __future__ import annotations
|
|
19
19
|
|
|
20
20
|
import logging
|
|
21
|
-
from typing import TYPE_CHECKING, Any,
|
|
21
|
+
from typing import TYPE_CHECKING, Any, Optional
|
|
22
22
|
from urllib.parse import parse_qs, urlencode, urlparse, urlunparse
|
|
23
23
|
|
|
24
24
|
import requests
|
|
@@ -60,9 +60,9 @@ class TokenAuth(Authentication):
|
|
|
60
60
|
* :attr:`~eodag.config.PluginConfig.type` (``str``) (**mandatory**): TokenAuth
|
|
61
61
|
* :attr:`~eodag.config.PluginConfig.auth_uri` (``str``) (**mandatory**): url used to fetch
|
|
62
62
|
the access token with user/password
|
|
63
|
-
* :attr:`~eodag.config.PluginConfig.headers` (``
|
|
63
|
+
* :attr:`~eodag.config.PluginConfig.headers` (``dict[str, str]``): Dictionary containing all
|
|
64
64
|
keys/value pairs that should be added to the headers
|
|
65
|
-
* :attr:`~eodag.config.PluginConfig.retrieve_headers` (``
|
|
65
|
+
* :attr:`~eodag.config.PluginConfig.retrieve_headers` (``dict[str, str]``): Dictionary containing all
|
|
66
66
|
keys/value pairs that should be added to the headers for token retrieve only
|
|
67
67
|
* :attr:`~eodag.config.PluginConfig.refresh_uri` (``str``) : url used to fetch the
|
|
68
68
|
access token with a refresh token
|
|
@@ -76,13 +76,13 @@ class TokenAuth(Authentication):
|
|
|
76
76
|
should be verified in the requests; default: ``True``
|
|
77
77
|
* :attr:`~eodag.config.PluginConfig.auth_error_code` (``int``): which error code is
|
|
78
78
|
returned in case of an authentication error
|
|
79
|
-
* :attr:`~eodag.config.PluginConfig.req_data` (``
|
|
79
|
+
* :attr:`~eodag.config.PluginConfig.req_data` (``dict[str, Any]``): if the credentials
|
|
80
80
|
should be sent as data in the post request, the json structure can be given in this parameter
|
|
81
81
|
* :attr:`~eodag.config.PluginConfig.retry_total` (``int``): :class:`urllib3.util.Retry` ``total`` parameter,
|
|
82
82
|
total number of retries to allow; default: ``3``
|
|
83
83
|
* :attr:`~eodag.config.PluginConfig.retry_backoff_factor` (``int``): :class:`urllib3.util.Retry`
|
|
84
84
|
``backoff_factor`` parameter, backoff factor to apply between attempts after the second try; default: ``2``
|
|
85
|
-
* :attr:`~eodag.config.PluginConfig.retry_status_forcelist` (``
|
|
85
|
+
* :attr:`~eodag.config.PluginConfig.retry_status_forcelist` (``list[int]``): :class:`urllib3.util.Retry`
|
|
86
86
|
``status_forcelist`` parameter, list of integer HTTP status codes that we should force a retry on; default:
|
|
87
87
|
``[401, 429, 500, 502, 503, 504]``
|
|
88
88
|
"""
|
|
@@ -200,7 +200,7 @@ class TokenAuth(Authentication):
|
|
|
200
200
|
headers = self.config.headers
|
|
201
201
|
|
|
202
202
|
# append headers to req if some are specified in config
|
|
203
|
-
req_kwargs:
|
|
203
|
+
req_kwargs: dict[str, Any] = {"headers": dict(headers, **USER_AGENT)}
|
|
204
204
|
ssl_verify = getattr(self.config, "ssl_verify", True)
|
|
205
205
|
|
|
206
206
|
if self.refresh_token:
|
|
@@ -260,7 +260,7 @@ class RequestsTokenAuth(AuthBase):
|
|
|
260
260
|
token: str,
|
|
261
261
|
where: str,
|
|
262
262
|
qs_key: Optional[str] = None,
|
|
263
|
-
headers: Optional[
|
|
263
|
+
headers: Optional[dict[str, str]] = None,
|
|
264
264
|
) -> None:
|
|
265
265
|
self.token = token
|
|
266
266
|
self.where = where
|
|
@@ -41,7 +41,7 @@ class OIDCTokenExchangeAuth(Authentication):
|
|
|
41
41
|
:param provider: provider name
|
|
42
42
|
:param config: Authentication plugin configuration:
|
|
43
43
|
|
|
44
|
-
* :attr:`~eodag.config.PluginConfig.subject` (``
|
|
44
|
+
* :attr:`~eodag.config.PluginConfig.subject` (``dict[str, Any]``) (**mandatory**):
|
|
45
45
|
The full :class:`~eodag.plugins.authentication.openid_connect.OIDCAuthorizationCodeFlowAuth` plugin
|
|
46
46
|
configuration used to retrieve subject token
|
|
47
47
|
* :attr:`~eodag.config.PluginConfig.subject_issuer` (``str``) (**mandatory**): Identifies
|
eodag/plugins/base.py
CHANGED
|
@@ -17,7 +17,7 @@
|
|
|
17
17
|
# limitations under the License.
|
|
18
18
|
from __future__ import annotations
|
|
19
19
|
|
|
20
|
-
from typing import TYPE_CHECKING, Any
|
|
20
|
+
from typing import TYPE_CHECKING, Any
|
|
21
21
|
|
|
22
22
|
from eodag.utils.exceptions import PluginNotFoundError
|
|
23
23
|
|
|
@@ -29,21 +29,21 @@ class EODAGPluginMount(type):
|
|
|
29
29
|
"""Plugin mount"""
|
|
30
30
|
|
|
31
31
|
def __init__(
|
|
32
|
-
cls, name: str, bases:
|
|
32
|
+
cls, name: str, bases: tuple[type, ...], attrs: dict[str, Any]
|
|
33
33
|
) -> None:
|
|
34
34
|
if not hasattr(cls, "plugins"):
|
|
35
35
|
# This branch only executes when processing the mount point itself.
|
|
36
36
|
# So, since this is a new plugin type, not an implementation, this
|
|
37
37
|
# class shouldn't be registered as a plugin. Instead, it sets up a
|
|
38
38
|
# list where plugins can be registered later.
|
|
39
|
-
cls.plugins:
|
|
39
|
+
cls.plugins: list[EODAGPluginMount] = []
|
|
40
40
|
else:
|
|
41
41
|
# This must be a plugin implementation, which should be registered.
|
|
42
42
|
# Simply appending it to the list is all that's needed to keep
|
|
43
43
|
# track of it later.
|
|
44
44
|
cls.plugins.append(cls)
|
|
45
45
|
|
|
46
|
-
def get_plugins(cls, *args: Any, **kwargs: Any) ->
|
|
46
|
+
def get_plugins(cls, *args: Any, **kwargs: Any) -> list[EODAGPluginMount]:
|
|
47
47
|
"""Get plugins"""
|
|
48
48
|
return [plugin(*args, **kwargs) for plugin in cls.plugins]
|
|
49
49
|
|
eodag/plugins/crunch/base.py
CHANGED
|
@@ -17,7 +17,7 @@
|
|
|
17
17
|
# limitations under the License
|
|
18
18
|
from __future__ import annotations
|
|
19
19
|
|
|
20
|
-
from typing import TYPE_CHECKING, Any,
|
|
20
|
+
from typing import TYPE_CHECKING, Any, Optional
|
|
21
21
|
|
|
22
22
|
from eodag.config import PluginConfig
|
|
23
23
|
from eodag.plugins.base import PluginTopic
|
|
@@ -32,12 +32,12 @@ class Crunch(PluginTopic):
|
|
|
32
32
|
:param config: Crunch configuration
|
|
33
33
|
"""
|
|
34
34
|
|
|
35
|
-
def __init__(self, config: Optional[
|
|
35
|
+
def __init__(self, config: Optional[dict[str, Any]]) -> None:
|
|
36
36
|
self.config = PluginConfig()
|
|
37
37
|
self.config.__dict__ = config if config is not None else {}
|
|
38
38
|
|
|
39
39
|
def proceed(
|
|
40
|
-
self, products:
|
|
41
|
-
) ->
|
|
40
|
+
self, products: list[EOProduct], **search_params: Any
|
|
41
|
+
) -> list[EOProduct]:
|
|
42
42
|
"""Implementation of how the results must be crunched"""
|
|
43
43
|
raise NotImplementedError
|
|
@@ -21,7 +21,7 @@ import datetime
|
|
|
21
21
|
import logging
|
|
22
22
|
import time
|
|
23
23
|
from datetime import datetime as dt
|
|
24
|
-
from typing import TYPE_CHECKING, Any
|
|
24
|
+
from typing import TYPE_CHECKING, Any
|
|
25
25
|
|
|
26
26
|
import dateutil.parser
|
|
27
27
|
from dateutil import tz
|
|
@@ -57,8 +57,8 @@ class FilterDate(Crunch):
|
|
|
57
57
|
return dateutil.parser.parse(start_date)
|
|
58
58
|
|
|
59
59
|
def proceed(
|
|
60
|
-
self, products:
|
|
61
|
-
) ->
|
|
60
|
+
self, products: list[EOProduct], **search_params: Any
|
|
61
|
+
) -> list[EOProduct]:
|
|
62
62
|
"""Execute crunch: Filter products between start and end dates.
|
|
63
63
|
|
|
64
64
|
:param products: A list of products resulting from a search
|
|
@@ -89,7 +89,7 @@ class FilterDate(Crunch):
|
|
|
89
89
|
if not filter_start and not filter_end:
|
|
90
90
|
return products
|
|
91
91
|
|
|
92
|
-
filtered:
|
|
92
|
+
filtered: list[EOProduct] = []
|
|
93
93
|
for product in products:
|
|
94
94
|
|
|
95
95
|
# product start date
|
|
@@ -20,7 +20,7 @@ from __future__ import annotations
|
|
|
20
20
|
import datetime
|
|
21
21
|
import logging
|
|
22
22
|
import time
|
|
23
|
-
from typing import TYPE_CHECKING, Any,
|
|
23
|
+
from typing import TYPE_CHECKING, Any, Union
|
|
24
24
|
|
|
25
25
|
import dateutil.parser
|
|
26
26
|
from shapely import geometry
|
|
@@ -54,14 +54,14 @@ class FilterLatestIntersect(Crunch):
|
|
|
54
54
|
return dateutil.parser.parse(start_date)
|
|
55
55
|
|
|
56
56
|
def proceed(
|
|
57
|
-
self, products:
|
|
58
|
-
) ->
|
|
57
|
+
self, products: list[EOProduct], **search_params: dict[str, Any]
|
|
58
|
+
) -> list[EOProduct]:
|
|
59
59
|
"""Execute crunch:
|
|
60
60
|
Filter latest products (the ones with a the highest start date) that intersect search extent.
|
|
61
61
|
|
|
62
62
|
:param products: A list of products resulting from a search
|
|
63
63
|
:param search_params: Search criteria that must contain ``geometry`` or ``geom`` parameters having value of
|
|
64
|
-
type :class:`shapely.geometry.base.BaseGeometry` or ``
|
|
64
|
+
type :class:`shapely.geometry.base.BaseGeometry` or ``dict[str, Any]``
|
|
65
65
|
:returns: The filtered products
|
|
66
66
|
"""
|
|
67
67
|
logger.debug("Start filtering for latest products")
|
|
@@ -69,9 +69,9 @@ class FilterLatestIntersect(Crunch):
|
|
|
69
69
|
return []
|
|
70
70
|
# Warning: May crash if startTimeFromAscendingNode is not in the appropriate format
|
|
71
71
|
products.sort(key=self.sort_product_by_start_date, reverse=True)
|
|
72
|
-
filtered:
|
|
72
|
+
filtered: list[EOProduct] = []
|
|
73
73
|
add_to_filtered = filtered.append
|
|
74
|
-
footprint: Union[
|
|
74
|
+
footprint: Union[dict[str, Any], BaseGeometry, Any] = search_params.get(
|
|
75
75
|
"geometry"
|
|
76
76
|
) or search_params.get("geom")
|
|
77
77
|
if not footprint:
|