eodag 3.0.0b3__py3-none-any.whl → 3.1.0b1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- eodag/api/core.py +292 -198
- eodag/api/product/_assets.py +6 -6
- eodag/api/product/_product.py +18 -18
- eodag/api/product/metadata_mapping.py +51 -14
- eodag/api/search_result.py +29 -3
- eodag/cli.py +57 -20
- eodag/config.py +413 -117
- eodag/plugins/apis/base.py +10 -4
- eodag/plugins/apis/ecmwf.py +49 -16
- eodag/plugins/apis/usgs.py +30 -7
- eodag/plugins/authentication/aws_auth.py +14 -5
- eodag/plugins/authentication/base.py +10 -1
- eodag/plugins/authentication/generic.py +14 -3
- eodag/plugins/authentication/header.py +12 -4
- eodag/plugins/authentication/keycloak.py +41 -22
- eodag/plugins/authentication/oauth.py +11 -1
- eodag/plugins/authentication/openid_connect.py +178 -163
- eodag/plugins/authentication/qsauth.py +12 -4
- eodag/plugins/authentication/sas_auth.py +19 -2
- eodag/plugins/authentication/token.py +93 -15
- eodag/plugins/authentication/token_exchange.py +19 -19
- eodag/plugins/crunch/base.py +4 -1
- eodag/plugins/crunch/filter_date.py +5 -2
- eodag/plugins/crunch/filter_latest_intersect.py +5 -4
- eodag/plugins/crunch/filter_latest_tpl_name.py +1 -1
- eodag/plugins/crunch/filter_overlap.py +5 -7
- eodag/plugins/crunch/filter_property.py +6 -6
- eodag/plugins/download/aws.py +50 -34
- eodag/plugins/download/base.py +41 -50
- eodag/plugins/download/creodias_s3.py +40 -2
- eodag/plugins/download/http.py +221 -195
- eodag/plugins/download/s3rest.py +25 -25
- eodag/plugins/manager.py +168 -23
- eodag/plugins/search/base.py +106 -39
- eodag/plugins/search/build_search_result.py +1065 -324
- eodag/plugins/search/cop_marine.py +112 -29
- eodag/plugins/search/creodias_s3.py +45 -24
- eodag/plugins/search/csw.py +41 -1
- eodag/plugins/search/data_request_search.py +109 -9
- eodag/plugins/search/qssearch.py +549 -257
- eodag/plugins/search/static_stac_search.py +20 -21
- eodag/resources/ext_product_types.json +1 -1
- eodag/resources/product_types.yml +577 -87
- eodag/resources/providers.yml +1619 -2776
- eodag/resources/stac.yml +3 -163
- eodag/resources/user_conf_template.yml +112 -97
- eodag/rest/config.py +1 -2
- eodag/rest/constants.py +0 -1
- eodag/rest/core.py +138 -98
- eodag/rest/errors.py +181 -0
- eodag/rest/server.py +55 -329
- eodag/rest/stac.py +93 -544
- eodag/rest/types/eodag_search.py +19 -8
- eodag/rest/types/queryables.py +6 -8
- eodag/rest/types/stac_search.py +11 -2
- eodag/rest/utils/__init__.py +3 -0
- eodag/types/__init__.py +71 -18
- eodag/types/download_args.py +3 -3
- eodag/types/queryables.py +180 -73
- eodag/types/search_args.py +3 -3
- eodag/types/whoosh.py +126 -0
- eodag/utils/__init__.py +147 -66
- eodag/utils/exceptions.py +47 -26
- eodag/utils/logging.py +37 -77
- eodag/utils/repr.py +65 -6
- eodag/utils/requests.py +11 -13
- eodag/utils/stac_reader.py +1 -1
- {eodag-3.0.0b3.dist-info → eodag-3.1.0b1.dist-info}/METADATA +80 -81
- eodag-3.1.0b1.dist-info/RECORD +108 -0
- {eodag-3.0.0b3.dist-info → eodag-3.1.0b1.dist-info}/WHEEL +1 -1
- {eodag-3.0.0b3.dist-info → eodag-3.1.0b1.dist-info}/entry_points.txt +4 -2
- eodag/resources/constraints/climate-dt.json +0 -13
- eodag/resources/constraints/extremes-dt.json +0 -8
- eodag/utils/constraints.py +0 -244
- eodag-3.0.0b3.dist-info/RECORD +0 -110
- {eodag-3.0.0b3.dist-info → eodag-3.1.0b1.dist-info}/LICENSE +0 -0
- {eodag-3.0.0b3.dist-info → eodag-3.1.0b1.dist-info}/top_level.txt +0 -0
|
@@ -28,7 +28,13 @@ from requests.auth import AuthBase
|
|
|
28
28
|
from urllib3 import Retry
|
|
29
29
|
|
|
30
30
|
from eodag.plugins.authentication.base import Authentication
|
|
31
|
-
from eodag.utils import
|
|
31
|
+
from eodag.utils import (
|
|
32
|
+
HTTP_REQ_TIMEOUT,
|
|
33
|
+
REQ_RETRY_BACKOFF_FACTOR,
|
|
34
|
+
REQ_RETRY_STATUS_FORCELIST,
|
|
35
|
+
REQ_RETRY_TOTAL,
|
|
36
|
+
USER_AGENT,
|
|
37
|
+
)
|
|
32
38
|
from eodag.utils.exceptions import AuthenticationError, MisconfiguredError, TimeOutError
|
|
33
39
|
|
|
34
40
|
if TYPE_CHECKING:
|
|
@@ -41,7 +47,45 @@ logger = logging.getLogger("eodag.authentication.token")
|
|
|
41
47
|
|
|
42
48
|
|
|
43
49
|
class TokenAuth(Authentication):
|
|
44
|
-
"""TokenAuth authentication plugin
|
|
50
|
+
"""TokenAuth authentication plugin - fetches a token which is added to search/download requests.
|
|
51
|
+
|
|
52
|
+
When using headers, if only :attr:`~eodag.config.PluginConfig.headers` is given, it will be used for both token
|
|
53
|
+
retrieve and authentication. If :attr:`~eodag.config.PluginConfig.retrieve_headers` is given, it will be used for
|
|
54
|
+
token retrieve only. If both are given, :attr:`~eodag.config.PluginConfig.retrieve_headers` will be used for token
|
|
55
|
+
retrieve and :attr:`~eodag.config.PluginConfig.headers` for authentication.
|
|
56
|
+
|
|
57
|
+
:param provider: provider name
|
|
58
|
+
:param config: Authentication plugin configuration:
|
|
59
|
+
|
|
60
|
+
* :attr:`~eodag.config.PluginConfig.type` (``str``) (**mandatory**): TokenAuth
|
|
61
|
+
* :attr:`~eodag.config.PluginConfig.auth_uri` (``str``) (**mandatory**): url used to fetch
|
|
62
|
+
the access token with user/password
|
|
63
|
+
* :attr:`~eodag.config.PluginConfig.headers` (``Dict[str, str]``): Dictionary containing all
|
|
64
|
+
keys/value pairs that should be added to the headers
|
|
65
|
+
* :attr:`~eodag.config.PluginConfig.retrieve_headers` (``Dict[str, str]``): Dictionary containing all
|
|
66
|
+
keys/value pairs that should be added to the headers for token retrieve only
|
|
67
|
+
* :attr:`~eodag.config.PluginConfig.refresh_uri` (``str``) : url used to fetch the
|
|
68
|
+
access token with a refresh token
|
|
69
|
+
* :attr:`~eodag.config.PluginConfig.token_type` (``str``): type of the token (``json``
|
|
70
|
+
or ``text``); default: ``text``
|
|
71
|
+
* :attr:`~eodag.config.PluginConfig.token_key` (``str``): (mandatory if token_type=json)
|
|
72
|
+
key to get the access token in the response to the token request
|
|
73
|
+
* :attr:`~eodag.config.PluginConfig.refresh_token_key` (``str``): key to get the refresh
|
|
74
|
+
token in the response to the token request
|
|
75
|
+
* :attr:`~eodag.config.PluginConfig.ssl_verify` (``bool``): if the ssl certificates
|
|
76
|
+
should be verified in the requests; default: ``True``
|
|
77
|
+
* :attr:`~eodag.config.PluginConfig.auth_error_code` (``int``): which error code is
|
|
78
|
+
returned in case of an authentication error
|
|
79
|
+
* :attr:`~eodag.config.PluginConfig.req_data` (``Dict[str, Any]``): if the credentials
|
|
80
|
+
should be sent as data in the post request, the json structure can be given in this parameter
|
|
81
|
+
* :attr:`~eodag.config.PluginConfig.retry_total` (``int``): :class:`urllib3.util.Retry` ``total`` parameter,
|
|
82
|
+
total number of retries to allow; default: ``3``
|
|
83
|
+
* :attr:`~eodag.config.PluginConfig.retry_backoff_factor` (``int``): :class:`urllib3.util.Retry`
|
|
84
|
+
``backoff_factor`` parameter, backoff factor to apply between attempts after the second try; default: ``2``
|
|
85
|
+
* :attr:`~eodag.config.PluginConfig.retry_status_forcelist` (``List[int]``): :class:`urllib3.util.Retry`
|
|
86
|
+
``status_forcelist`` parameter, list of integer HTTP status codes that we should force a retry on; default:
|
|
87
|
+
``[401, 429, 500, 502, 503, 504]``
|
|
88
|
+
"""
|
|
45
89
|
|
|
46
90
|
def __init__(self, provider: str, config: PluginConfig) -> None:
|
|
47
91
|
super(TokenAuth, self).__init__(provider, config)
|
|
@@ -56,11 +100,29 @@ class TokenAuth(Authentication):
|
|
|
56
100
|
self.config.auth_uri = self.config.auth_uri.format(
|
|
57
101
|
**self.config.credentials
|
|
58
102
|
)
|
|
59
|
-
|
|
103
|
+
|
|
104
|
+
# Format headers if needed with values from the credentials. Note:
|
|
105
|
+
# if only 'headers' is given, it will be used for both token retrieve and authentication.
|
|
106
|
+
# if 'retrieve_headers' is given, it will be used for token retrieve only.
|
|
107
|
+
# if both are given, 'retrieve_headers' will be used for token retrieve and 'headers' for authentication.
|
|
108
|
+
|
|
109
|
+
# If the authentication headers are undefined or None: use an empty dict.
|
|
110
|
+
# And don't format '{token}' now, it will be done later.
|
|
111
|
+
raw_headers = getattr(self.config, "headers", None) or {}
|
|
60
112
|
self.config.headers = {
|
|
61
113
|
header: value.format(**{"token": "{token}", **self.config.credentials})
|
|
62
|
-
for header, value in
|
|
114
|
+
for header, value in raw_headers.items()
|
|
63
115
|
}
|
|
116
|
+
|
|
117
|
+
# If the retrieve headers are undefined, their attribute must not be set in self.config.
|
|
118
|
+
# If they are defined but empty, use an empty dict instead of None.
|
|
119
|
+
if hasattr(self.config, "retrieve_headers"):
|
|
120
|
+
raw_retrieve_headers = self.config.retrieve_headers or {}
|
|
121
|
+
self.config.retrieve_headers = {
|
|
122
|
+
header: value.format(**self.config.credentials)
|
|
123
|
+
for header, value in raw_retrieve_headers.items()
|
|
124
|
+
}
|
|
125
|
+
|
|
64
126
|
except KeyError as e:
|
|
65
127
|
raise MisconfiguredError(
|
|
66
128
|
f"Missing credentials inputs for provider {self.provider}: {e}"
|
|
@@ -89,14 +151,15 @@ class TokenAuth(Authentication):
|
|
|
89
151
|
and e.response.status_code in auth_errors
|
|
90
152
|
):
|
|
91
153
|
raise AuthenticationError(
|
|
92
|
-
f"
|
|
93
|
-
f"
|
|
94
|
-
|
|
154
|
+
f"Please check your credentials for {self.provider}.",
|
|
155
|
+
f"HTTP Error {e.response.status_code} returned.",
|
|
156
|
+
response_text,
|
|
157
|
+
) from e
|
|
95
158
|
# other error
|
|
96
159
|
else:
|
|
97
160
|
raise AuthenticationError(
|
|
98
|
-
|
|
99
|
-
)
|
|
161
|
+
"Could no get authentication token", str(e), response_text
|
|
162
|
+
) from e
|
|
100
163
|
else:
|
|
101
164
|
if getattr(self.config, "token_type", "text") == "json":
|
|
102
165
|
token = response.json()[self.config.token_key]
|
|
@@ -116,16 +179,29 @@ class TokenAuth(Authentication):
|
|
|
116
179
|
self,
|
|
117
180
|
session: requests.Session,
|
|
118
181
|
) -> requests.Response:
|
|
182
|
+
retry_total = getattr(self.config, "retry_total", REQ_RETRY_TOTAL)
|
|
183
|
+
retry_backoff_factor = getattr(
|
|
184
|
+
self.config, "retry_backoff_factor", REQ_RETRY_BACKOFF_FACTOR
|
|
185
|
+
)
|
|
186
|
+
retry_status_forcelist = getattr(
|
|
187
|
+
self.config, "retry_status_forcelist", REQ_RETRY_STATUS_FORCELIST
|
|
188
|
+
)
|
|
189
|
+
|
|
119
190
|
retries = Retry(
|
|
120
|
-
total=
|
|
121
|
-
backoff_factor=
|
|
122
|
-
status_forcelist=
|
|
191
|
+
total=retry_total,
|
|
192
|
+
backoff_factor=retry_backoff_factor,
|
|
193
|
+
status_forcelist=retry_status_forcelist,
|
|
123
194
|
)
|
|
124
195
|
|
|
196
|
+
# Use the headers for retrieval if defined, else the headers for authentication
|
|
197
|
+
try:
|
|
198
|
+
headers = self.config.retrieve_headers
|
|
199
|
+
except AttributeError:
|
|
200
|
+
headers = self.config.headers
|
|
201
|
+
|
|
125
202
|
# append headers to req if some are specified in config
|
|
126
|
-
req_kwargs: Dict[str, Any] = {
|
|
127
|
-
|
|
128
|
-
}
|
|
203
|
+
req_kwargs: Dict[str, Any] = {"headers": dict(headers, **USER_AGENT)}
|
|
204
|
+
ssl_verify = getattr(self.config, "ssl_verify", True)
|
|
129
205
|
|
|
130
206
|
if self.refresh_token:
|
|
131
207
|
logger.debug("fetching access token with refresh token")
|
|
@@ -135,6 +211,7 @@ class TokenAuth(Authentication):
|
|
|
135
211
|
self.config.refresh_uri,
|
|
136
212
|
data={"refresh_token": self.refresh_token},
|
|
137
213
|
timeout=HTTP_REQ_TIMEOUT,
|
|
214
|
+
verify=ssl_verify,
|
|
138
215
|
**req_kwargs,
|
|
139
216
|
)
|
|
140
217
|
response.raise_for_status()
|
|
@@ -170,6 +247,7 @@ class TokenAuth(Authentication):
|
|
|
170
247
|
method=method,
|
|
171
248
|
url=self.config.auth_uri,
|
|
172
249
|
timeout=HTTP_REQ_TIMEOUT,
|
|
250
|
+
verify=ssl_verify,
|
|
173
251
|
**req_kwargs,
|
|
174
252
|
)
|
|
175
253
|
|
|
@@ -38,25 +38,23 @@ class OIDCTokenExchangeAuth(Authentication):
|
|
|
38
38
|
"""Token exchange implementation using
|
|
39
39
|
:class:`~eodag.plugins.authentication.openid_connect.OIDCAuthorizationCodeFlowAuth` token as subject.
|
|
40
40
|
|
|
41
|
-
|
|
41
|
+
:param provider: provider name
|
|
42
|
+
:param config: Authentication plugin configuration:
|
|
43
|
+
|
|
44
|
+
* :attr:`~eodag.config.PluginConfig.subject` (``Dict[str, Any]``) (**mandatory**):
|
|
45
|
+
The full :class:`~eodag.plugins.authentication.openid_connect.OIDCAuthorizationCodeFlowAuth` plugin
|
|
46
|
+
configuration used to retrieve subject token
|
|
47
|
+
* :attr:`~eodag.config.PluginConfig.subject_issuer` (``str``) (**mandatory**): Identifies
|
|
48
|
+
the issuer of the subject_token
|
|
49
|
+
* :attr:`~eodag.config.PluginConfig.token_uri` (``str``) (**mandatory**): The url to
|
|
50
|
+
query to get the authorized token
|
|
51
|
+
* :attr:`~eodag.config.PluginConfig.client_id` (``str``) (**mandatory**): The OIDC
|
|
52
|
+
provider's client ID of the eodag provider
|
|
53
|
+
* :attr:`~eodag.config.PluginConfig.audience` (``str``) (**mandatory**): This parameter
|
|
54
|
+
specifies the target client you want the new token minted for.
|
|
55
|
+
* :attr:`~eodag.config.PluginConfig.token_key` (``str``) (**mandatory**): The key
|
|
56
|
+
pointing to the token in the json response to the POST request to the token server
|
|
42
57
|
|
|
43
|
-
# (mandatory) The full OIDCAuthorizationCodeFlowAuth plugin configuration used to retrieve subject token
|
|
44
|
-
subject:
|
|
45
|
-
|
|
46
|
-
# (mandatory) Identifies the issuer of the subject_token
|
|
47
|
-
subject_issuer:
|
|
48
|
-
|
|
49
|
-
# (mandatory) The url to query to get the authorized token
|
|
50
|
-
token_uri:
|
|
51
|
-
|
|
52
|
-
# (mandatory) The OIDC provider's client ID of the eodag provider
|
|
53
|
-
client_id:
|
|
54
|
-
|
|
55
|
-
# (mandatory) This parameter specifies the target client you want the new token minted for.
|
|
56
|
-
audience:
|
|
57
|
-
|
|
58
|
-
# (mandatory) The key pointing to the token in the json response to the POST request to the token server
|
|
59
|
-
token_key:
|
|
60
58
|
"""
|
|
61
59
|
|
|
62
60
|
GRANT_TYPE = "urn:ietf:params:oauth:grant-type:token-exchange"
|
|
@@ -71,7 +69,7 @@ class OIDCTokenExchangeAuth(Authentication):
|
|
|
71
69
|
]
|
|
72
70
|
|
|
73
71
|
def __init__(self, provider: str, config: PluginConfig) -> None:
|
|
74
|
-
super(
|
|
72
|
+
super().__init__(provider, config)
|
|
75
73
|
for required_key in self.REQUIRED_KEYS:
|
|
76
74
|
if getattr(self.config, required_key, None) is None:
|
|
77
75
|
raise MisconfiguredError(
|
|
@@ -100,12 +98,14 @@ class OIDCTokenExchangeAuth(Authentication):
|
|
|
100
98
|
"audience": self.config.audience,
|
|
101
99
|
}
|
|
102
100
|
logger.debug("Getting target auth token")
|
|
101
|
+
ssl_verify = getattr(self.config, "ssl_verify", True)
|
|
103
102
|
try:
|
|
104
103
|
auth_response = self.subject.session.post(
|
|
105
104
|
self.config.token_uri,
|
|
106
105
|
data=auth_data,
|
|
107
106
|
headers=USER_AGENT,
|
|
108
107
|
timeout=HTTP_REQ_TIMEOUT,
|
|
108
|
+
verify=ssl_verify,
|
|
109
109
|
)
|
|
110
110
|
auth_response.raise_for_status()
|
|
111
111
|
except requests.exceptions.Timeout as exc:
|
eodag/plugins/crunch/base.py
CHANGED
|
@@ -37,10 +37,13 @@ logger = logging.getLogger("eodag.crunch.date")
|
|
|
37
37
|
class FilterDate(Crunch):
|
|
38
38
|
"""FilterDate cruncher: filter products by date
|
|
39
39
|
|
|
40
|
+
Allows to filter out products that are older than a start date (optional) or more recent than an end date
|
|
41
|
+
(optional).
|
|
42
|
+
|
|
40
43
|
:param config: Crunch configuration, may contain :
|
|
41
44
|
|
|
42
|
-
|
|
43
|
-
|
|
45
|
+
* ``start`` (``str``): start sensing time in iso format
|
|
46
|
+
* ``end`` (``str``): end sensing time in iso format
|
|
44
47
|
"""
|
|
45
48
|
|
|
46
49
|
@staticmethod
|
|
@@ -39,7 +39,8 @@ logger = logging.getLogger("eodag.crunch.latest_intersect")
|
|
|
39
39
|
class FilterLatestIntersect(Crunch):
|
|
40
40
|
"""FilterLatestIntersect cruncher
|
|
41
41
|
|
|
42
|
-
Filter latest products (the ones with a the highest start date) that intersect search extent
|
|
42
|
+
Filter latest products (the ones with a the highest start date) that intersect search extent;
|
|
43
|
+
The configuration for this plugin is an empty dict
|
|
43
44
|
"""
|
|
44
45
|
|
|
45
46
|
@staticmethod
|
|
@@ -53,14 +54,14 @@ class FilterLatestIntersect(Crunch):
|
|
|
53
54
|
return dateutil.parser.parse(start_date)
|
|
54
55
|
|
|
55
56
|
def proceed(
|
|
56
|
-
self, products: List[EOProduct], **search_params: Any
|
|
57
|
+
self, products: List[EOProduct], **search_params: Dict[str, Any]
|
|
57
58
|
) -> List[EOProduct]:
|
|
58
59
|
"""Execute crunch:
|
|
59
60
|
Filter latest products (the ones with a the highest start date) that intersect search extent.
|
|
60
61
|
|
|
61
62
|
:param products: A list of products resulting from a search
|
|
62
|
-
:param search_params: Search criteria that must contain
|
|
63
|
-
|
|
63
|
+
:param search_params: Search criteria that must contain ``geometry`` or ``geom`` parameters having value of
|
|
64
|
+
type :class:`shapely.geometry.base.BaseGeometry` or ``Dict[str, Any]``
|
|
64
65
|
:returns: The filtered products
|
|
65
66
|
"""
|
|
66
67
|
logger.debug("Start filtering for latest products")
|
|
@@ -37,7 +37,7 @@ class FilterLatestByName(Crunch):
|
|
|
37
37
|
|
|
38
38
|
:param config: Crunch configuration, must contain :
|
|
39
39
|
|
|
40
|
-
|
|
40
|
+
* ``name_pattern`` (``str``) (**mandatory**): product name pattern
|
|
41
41
|
"""
|
|
42
42
|
|
|
43
43
|
NAME_PATTERN_CONSTRAINT = re.compile(r"\(\?P<tileid>\\d\{6\}\)")
|
|
@@ -40,14 +40,12 @@ class FilterOverlap(Crunch):
|
|
|
40
40
|
|
|
41
41
|
Filter products, retaining only those that are overlapping with the search_extent
|
|
42
42
|
|
|
43
|
-
:param config: Crunch configuration
|
|
43
|
+
:param config: Crunch configuration may contain the following parameters which are mutually exclusive:
|
|
44
44
|
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
These configuration parameters are mutually exclusive.
|
|
45
|
+
* ``minimum_overlap`` (``Union[float, str]``): minimal overlap percentage; default: ``"0"``
|
|
46
|
+
* ``contains`` (``bool``): ``True`` if product geometry contains the search area; default: ``False``
|
|
47
|
+
* ``intersects`` (``bool``): ``True`` if product geometry intersects the search area; default: ``False``
|
|
48
|
+
* ``within`` (``bool``): ``True`` if product geometry is within the search area; default: ``False``
|
|
51
49
|
"""
|
|
52
50
|
|
|
53
51
|
def proceed(
|
|
@@ -34,10 +34,11 @@ class FilterProperty(Crunch):
|
|
|
34
34
|
|
|
35
35
|
Filter products, retaining only those whose property match criteria
|
|
36
36
|
|
|
37
|
-
:param config: Crunch configuration,
|
|
37
|
+
:param config: Crunch configuration, must contain :
|
|
38
38
|
|
|
39
|
-
|
|
40
|
-
|
|
39
|
+
* ``<property>`` ``(Any)`` (**mandatory**): property key from ``product.properties``, associated to its filter
|
|
40
|
+
value
|
|
41
|
+
* ``operator`` (``str``): Operator used for filtering (one of ``lt,le,eq,ne,ge,gt``). Default is ``eq``
|
|
41
42
|
"""
|
|
42
43
|
|
|
43
44
|
def proceed(
|
|
@@ -77,10 +78,9 @@ class FilterProperty(Crunch):
|
|
|
77
78
|
for product in products:
|
|
78
79
|
if property_key not in product.properties.keys():
|
|
79
80
|
logger.warning(
|
|
80
|
-
"
|
|
81
|
-
property_key,
|
|
81
|
+
f"{property_key} not found in {product}.properties, product skipped",
|
|
82
82
|
)
|
|
83
|
-
|
|
83
|
+
continue
|
|
84
84
|
if operator_method(product.properties[property_key], property_value):
|
|
85
85
|
add_to_filtered(product)
|
|
86
86
|
|
eodag/plugins/download/aws.py
CHANGED
|
@@ -70,6 +70,7 @@ from eodag.utils.exceptions import (
|
|
|
70
70
|
AuthenticationError,
|
|
71
71
|
DownloadError,
|
|
72
72
|
MisconfiguredError,
|
|
73
|
+
NoMatchingProductType,
|
|
73
74
|
NotAvailableError,
|
|
74
75
|
TimeOutError,
|
|
75
76
|
)
|
|
@@ -216,12 +217,28 @@ class AwsDownload(Download):
|
|
|
216
217
|
:param provider: provider name
|
|
217
218
|
:param config: Download plugin configuration:
|
|
218
219
|
|
|
219
|
-
*
|
|
220
|
-
*
|
|
221
|
-
|
|
222
|
-
|
|
223
|
-
*
|
|
224
|
-
|
|
220
|
+
* :attr:`~eodag.config.PluginConfig.type` (``str``) (**mandatory**): AwsDownload
|
|
221
|
+
* :attr:`~eodag.config.PluginConfig.base_uri` (``str``) (**mandatory**): s3 endpoint url
|
|
222
|
+
* :attr:`~eodag.config.PluginConfig.requester_pays` (``bool``): whether download is done
|
|
223
|
+
from a requester-pays bucket or not; default: ``False``
|
|
224
|
+
* :attr:`~eodag.config.PluginConfig.flatten_top_dirs` (``bool``): if the directory structure
|
|
225
|
+
should be flattened; default: ``True``
|
|
226
|
+
* :attr:`~eodag.config.PluginConfig.ignore_assets` (``bool``): ignore assets and download
|
|
227
|
+
using ``downloadLink``; default: ``False``
|
|
228
|
+
* :attr:`~eodag.config.PluginConfig.ssl_verify` (``bool``): if the ssl certificates should
|
|
229
|
+
be verified in requests; default: ``True``
|
|
230
|
+
* :attr:`~eodag.config.PluginConfig.bucket_path_level` (``int``): at which level of the
|
|
231
|
+
path part of the url the bucket can be found; If no bucket_path_level is given, the bucket
|
|
232
|
+
is taken from the first element of the netloc part.
|
|
233
|
+
* :attr:`~eodag.config.PluginConfig.products` (``Dict[str, Dict[str, Any]``): product type
|
|
234
|
+
specific config; the keys are the product types, the values are dictionaries which can contain the keys:
|
|
235
|
+
|
|
236
|
+
* **default_bucket** (``str``): bucket where the product type can be found
|
|
237
|
+
* **complementary_url_key** (``str``): keys to add additional urls
|
|
238
|
+
* **build_safe** (``bool``): if a SAFE (Standard Archive Format for Europe) product should
|
|
239
|
+
be created; used for Sentinel products; default: False
|
|
240
|
+
* **fetch_metadata** (``Dict[str, Any]``): config for metadata to be fetched for the SAFE product
|
|
241
|
+
|
|
225
242
|
"""
|
|
226
243
|
|
|
227
244
|
def __init__(self, provider: str, config: PluginConfig) -> None:
|
|
@@ -234,8 +251,8 @@ class AwsDownload(Download):
|
|
|
234
251
|
product: EOProduct,
|
|
235
252
|
auth: Optional[Union[AuthBase, Dict[str, str]]] = None,
|
|
236
253
|
progress_callback: Optional[ProgressCallback] = None,
|
|
237
|
-
wait:
|
|
238
|
-
timeout:
|
|
254
|
+
wait: float = DEFAULT_DOWNLOAD_WAIT,
|
|
255
|
+
timeout: float = DEFAULT_DOWNLOAD_TIMEOUT,
|
|
239
256
|
**kwargs: Unpack[DownloadConf],
|
|
240
257
|
) -> Optional[str]:
|
|
241
258
|
"""Download method for AWS S3 API.
|
|
@@ -465,7 +482,8 @@ class AwsDownload(Download):
|
|
|
465
482
|
ignore_assets: Optional[bool] = False,
|
|
466
483
|
) -> List[Tuple[str, Optional[str]]]:
|
|
467
484
|
"""
|
|
468
|
-
|
|
485
|
+
Retrieves the bucket names and path prefixes for the assets
|
|
486
|
+
|
|
469
487
|
:param product: product for which the assets shall be downloaded
|
|
470
488
|
:param asset_filter: text for which the assets should be filtered
|
|
471
489
|
:param ignore_assets: if product instead of individual assets should be used
|
|
@@ -611,19 +629,20 @@ class AwsDownload(Download):
|
|
|
611
629
|
raise NotAvailableError(
|
|
612
630
|
rf"No file basename matching re.fullmatch(r'{asset_filter}') was found in {product.remote_location}"
|
|
613
631
|
)
|
|
632
|
+
|
|
633
|
+
if not unique_product_chunks:
|
|
634
|
+
raise NoMatchingProductType("No product found to download.")
|
|
635
|
+
|
|
614
636
|
return unique_product_chunks
|
|
615
637
|
|
|
616
638
|
def _raise_if_auth_error(self, exception: ClientError) -> None:
|
|
617
639
|
"""Raises an error if given exception is an authentication error"""
|
|
618
|
-
err = exception.response["Error"]
|
|
640
|
+
err = cast(Dict[str, str], exception.response["Error"])
|
|
619
641
|
if err["Code"] in AWS_AUTH_ERROR_MESSAGES and "key" in err["Message"].lower():
|
|
620
642
|
raise AuthenticationError(
|
|
621
|
-
"
|
|
622
|
-
|
|
623
|
-
|
|
624
|
-
err["Message"],
|
|
625
|
-
self.provider,
|
|
626
|
-
)
|
|
643
|
+
f"Please check your credentials for {self.provider}.",
|
|
644
|
+
f"HTTP Error {exception.response['ResponseMetadata']['HTTPStatusCode']} returned.",
|
|
645
|
+
err["Code"] + ": " + err["Message"],
|
|
627
646
|
)
|
|
628
647
|
|
|
629
648
|
def _stream_download_dict(
|
|
@@ -631,12 +650,12 @@ class AwsDownload(Download):
|
|
|
631
650
|
product: EOProduct,
|
|
632
651
|
auth: Optional[Union[AuthBase, Dict[str, str]]] = None,
|
|
633
652
|
progress_callback: Optional[ProgressCallback] = None,
|
|
634
|
-
wait:
|
|
635
|
-
timeout:
|
|
653
|
+
wait: float = DEFAULT_DOWNLOAD_WAIT,
|
|
654
|
+
timeout: float = DEFAULT_DOWNLOAD_TIMEOUT,
|
|
636
655
|
**kwargs: Unpack[DownloadConf],
|
|
637
656
|
) -> StreamResponse:
|
|
638
657
|
r"""
|
|
639
|
-
Returns
|
|
658
|
+
Returns dictionary of :class:`~fastapi.responses.StreamingResponse` keyword-arguments.
|
|
640
659
|
It contains a generator to streamed download chunks and the response headers.
|
|
641
660
|
|
|
642
661
|
:param product: The EO product to download
|
|
@@ -649,7 +668,7 @@ class AwsDownload(Download):
|
|
|
649
668
|
and `dl_url_params` (dict) can be provided as additional kwargs
|
|
650
669
|
and will override any other values defined in a configuration
|
|
651
670
|
file or with environment variables.
|
|
652
|
-
:returns:
|
|
671
|
+
:returns: Dictionary of :class:`~fastapi.responses.StreamingResponse` keyword-arguments
|
|
653
672
|
"""
|
|
654
673
|
if progress_callback is None:
|
|
655
674
|
logger.info(
|
|
@@ -713,12 +732,12 @@ class AwsDownload(Download):
|
|
|
713
732
|
else sanitize(product.properties.get("id", "download"))
|
|
714
733
|
)
|
|
715
734
|
|
|
716
|
-
if len(assets_values)
|
|
735
|
+
if len(assets_values) <= 1:
|
|
717
736
|
first_chunks_tuple = next(chunks_tuples)
|
|
718
737
|
# update headers
|
|
719
738
|
filename = os.path.basename(list(unique_product_chunks)[0].key)
|
|
720
739
|
headers = {"content-disposition": f"attachment; filename={filename}"}
|
|
721
|
-
if assets_values[0].get("type", None):
|
|
740
|
+
if assets_values and assets_values[0].get("type", None):
|
|
722
741
|
headers["content-type"] = assets_values[0]["type"]
|
|
723
742
|
|
|
724
743
|
return StreamResponse(
|
|
@@ -751,7 +770,6 @@ class AwsDownload(Download):
|
|
|
751
770
|
product_chunk: Any, progress_callback: ProgressCallback
|
|
752
771
|
) -> Any:
|
|
753
772
|
try:
|
|
754
|
-
|
|
755
773
|
chunk_start = 0
|
|
756
774
|
chunk_end = chunk_start + chunk_size - 1
|
|
757
775
|
|
|
@@ -782,7 +800,6 @@ class AwsDownload(Download):
|
|
|
782
800
|
common_path = self._get_commonpath(
|
|
783
801
|
product, unique_product_chunks, build_safe
|
|
784
802
|
)
|
|
785
|
-
|
|
786
803
|
for product_chunk in unique_product_chunks:
|
|
787
804
|
try:
|
|
788
805
|
chunk_rel_path = self.get_chunk_dest_path(
|
|
@@ -800,8 +817,7 @@ class AwsDownload(Download):
|
|
|
800
817
|
# out of SAFE format chunk
|
|
801
818
|
logger.warning(e)
|
|
802
819
|
continue
|
|
803
|
-
|
|
804
|
-
if len(assets_values) == 1:
|
|
820
|
+
if len(assets_values) <= 1:
|
|
805
821
|
yield from get_chunk_parts(product_chunk, progress_callback)
|
|
806
822
|
else:
|
|
807
823
|
yield (
|
|
@@ -829,7 +845,7 @@ class AwsDownload(Download):
|
|
|
829
845
|
|
|
830
846
|
:param bucket_name: Bucket containg objects
|
|
831
847
|
:param prefix: Prefix used to try auth
|
|
832
|
-
:param auth_dict:
|
|
848
|
+
:param auth_dict: Dictionary containing authentication keys
|
|
833
849
|
:returns: The rasterio environement variables
|
|
834
850
|
"""
|
|
835
851
|
if self.s3_session is not None:
|
|
@@ -857,7 +873,7 @@ class AwsDownload(Download):
|
|
|
857
873
|
:param bucket_name: Bucket containg objects
|
|
858
874
|
:param prefix: Prefix used to filter objects on auth try
|
|
859
875
|
(not used to filter returned objects)
|
|
860
|
-
:param auth_dict:
|
|
876
|
+
:param auth_dict: Dictionary containing authentication keys
|
|
861
877
|
:returns: The boto3 authenticated objects
|
|
862
878
|
"""
|
|
863
879
|
auth_methods: List[
|
|
@@ -901,7 +917,7 @@ class AwsDownload(Download):
|
|
|
901
917
|
"""Auth strategy using no-sign-request"""
|
|
902
918
|
|
|
903
919
|
s3_resource = boto3.resource(
|
|
904
|
-
service_name="s3", endpoint_url=getattr(self.config, "
|
|
920
|
+
service_name="s3", endpoint_url=getattr(self.config, "s3_endpoint", None)
|
|
905
921
|
)
|
|
906
922
|
s3_resource.meta.client.meta.events.register(
|
|
907
923
|
"choose-signer.s3.*", disable_signing
|
|
@@ -919,7 +935,7 @@ class AwsDownload(Download):
|
|
|
919
935
|
s3_session = boto3.session.Session(profile_name=auth_dict["profile_name"])
|
|
920
936
|
s3_resource = s3_session.resource(
|
|
921
937
|
service_name="s3",
|
|
922
|
-
endpoint_url=getattr(self.config, "
|
|
938
|
+
endpoint_url=getattr(self.config, "s3_endpoint", None),
|
|
923
939
|
)
|
|
924
940
|
if self.requester_pays:
|
|
925
941
|
objects = s3_resource.Bucket(bucket_name).objects.filter(
|
|
@@ -958,7 +974,7 @@ class AwsDownload(Download):
|
|
|
958
974
|
s3_session = boto3.session.Session(**s3_session_kwargs)
|
|
959
975
|
s3_resource = s3_session.resource(
|
|
960
976
|
service_name="s3",
|
|
961
|
-
endpoint_url=getattr(self.config, "
|
|
977
|
+
endpoint_url=getattr(self.config, "s3_endpoint", None),
|
|
962
978
|
)
|
|
963
979
|
if self.requester_pays:
|
|
964
980
|
objects = s3_resource.Bucket(bucket_name).objects.filter(
|
|
@@ -979,7 +995,7 @@ class AwsDownload(Download):
|
|
|
979
995
|
|
|
980
996
|
s3_session = boto3.session.Session()
|
|
981
997
|
s3_resource = s3_session.resource(
|
|
982
|
-
service_name="s3", endpoint_url=getattr(self.config, "
|
|
998
|
+
service_name="s3", endpoint_url=getattr(self.config, "s3_endpoint", None)
|
|
983
999
|
)
|
|
984
1000
|
if self.requester_pays:
|
|
985
1001
|
objects = s3_resource.Bucket(bucket_name).objects.filter(
|
|
@@ -1313,8 +1329,8 @@ class AwsDownload(Download):
|
|
|
1313
1329
|
auth: Optional[Union[AuthBase, Dict[str, str]]] = None,
|
|
1314
1330
|
downloaded_callback: Optional[DownloadedCallback] = None,
|
|
1315
1331
|
progress_callback: Optional[ProgressCallback] = None,
|
|
1316
|
-
wait:
|
|
1317
|
-
timeout:
|
|
1332
|
+
wait: float = DEFAULT_DOWNLOAD_WAIT,
|
|
1333
|
+
timeout: float = DEFAULT_DOWNLOAD_TIMEOUT,
|
|
1318
1334
|
**kwargs: Unpack[DownloadConf],
|
|
1319
1335
|
) -> List[str]:
|
|
1320
1336
|
"""
|