eodag 3.0.0b3__py3-none-any.whl → 3.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- eodag/api/core.py +347 -247
- eodag/api/product/_assets.py +44 -15
- eodag/api/product/_product.py +58 -47
- eodag/api/product/drivers/__init__.py +81 -4
- eodag/api/product/drivers/base.py +65 -4
- eodag/api/product/drivers/generic.py +65 -0
- eodag/api/product/drivers/sentinel1.py +97 -0
- eodag/api/product/drivers/sentinel2.py +95 -0
- eodag/api/product/metadata_mapping.py +129 -93
- eodag/api/search_result.py +28 -12
- eodag/cli.py +61 -24
- eodag/config.py +457 -167
- eodag/plugins/apis/base.py +10 -4
- eodag/plugins/apis/ecmwf.py +53 -23
- eodag/plugins/apis/usgs.py +41 -17
- eodag/plugins/authentication/aws_auth.py +30 -18
- eodag/plugins/authentication/base.py +14 -3
- eodag/plugins/authentication/generic.py +14 -3
- eodag/plugins/authentication/header.py +14 -6
- eodag/plugins/authentication/keycloak.py +44 -25
- eodag/plugins/authentication/oauth.py +18 -4
- eodag/plugins/authentication/openid_connect.py +192 -171
- eodag/plugins/authentication/qsauth.py +12 -4
- eodag/plugins/authentication/sas_auth.py +22 -5
- eodag/plugins/authentication/token.py +95 -17
- eodag/plugins/authentication/token_exchange.py +19 -19
- eodag/plugins/base.py +4 -4
- eodag/plugins/crunch/base.py +8 -5
- eodag/plugins/crunch/filter_date.py +9 -6
- eodag/plugins/crunch/filter_latest_intersect.py +9 -8
- eodag/plugins/crunch/filter_latest_tpl_name.py +8 -8
- eodag/plugins/crunch/filter_overlap.py +9 -11
- eodag/plugins/crunch/filter_property.py +10 -10
- eodag/plugins/download/aws.py +181 -105
- eodag/plugins/download/base.py +49 -67
- eodag/plugins/download/creodias_s3.py +40 -2
- eodag/plugins/download/http.py +247 -223
- eodag/plugins/download/s3rest.py +29 -28
- eodag/plugins/manager.py +176 -41
- eodag/plugins/search/__init__.py +6 -5
- eodag/plugins/search/base.py +123 -60
- eodag/plugins/search/build_search_result.py +1046 -355
- eodag/plugins/search/cop_marine.py +132 -39
- eodag/plugins/search/creodias_s3.py +19 -68
- eodag/plugins/search/csw.py +48 -8
- eodag/plugins/search/data_request_search.py +124 -23
- eodag/plugins/search/qssearch.py +531 -310
- eodag/plugins/search/stac_list_assets.py +85 -0
- eodag/plugins/search/static_stac_search.py +23 -24
- eodag/resources/ext_product_types.json +1 -1
- eodag/resources/product_types.yml +1295 -355
- eodag/resources/providers.yml +1819 -3010
- eodag/resources/stac.yml +3 -163
- eodag/resources/stac_api.yml +2 -2
- eodag/resources/user_conf_template.yml +115 -99
- eodag/rest/cache.py +2 -2
- eodag/rest/config.py +3 -4
- eodag/rest/constants.py +0 -1
- eodag/rest/core.py +157 -117
- eodag/rest/errors.py +181 -0
- eodag/rest/server.py +57 -339
- eodag/rest/stac.py +133 -581
- eodag/rest/types/collections_search.py +3 -3
- eodag/rest/types/eodag_search.py +41 -30
- eodag/rest/types/queryables.py +42 -32
- eodag/rest/types/stac_search.py +15 -16
- eodag/rest/utils/__init__.py +14 -21
- eodag/rest/utils/cql_evaluate.py +6 -6
- eodag/rest/utils/rfc3339.py +2 -2
- eodag/types/__init__.py +153 -32
- eodag/types/bbox.py +2 -2
- eodag/types/download_args.py +4 -4
- eodag/types/queryables.py +183 -73
- eodag/types/search_args.py +6 -6
- eodag/types/whoosh.py +127 -3
- eodag/utils/__init__.py +228 -106
- eodag/utils/exceptions.py +47 -26
- eodag/utils/import_system.py +2 -2
- eodag/utils/logging.py +37 -77
- eodag/utils/repr.py +65 -6
- eodag/utils/requests.py +13 -15
- eodag/utils/rest.py +2 -2
- eodag/utils/s3.py +231 -0
- eodag/utils/stac_reader.py +11 -11
- {eodag-3.0.0b3.dist-info → eodag-3.1.0.dist-info}/METADATA +81 -81
- eodag-3.1.0.dist-info/RECORD +113 -0
- {eodag-3.0.0b3.dist-info → eodag-3.1.0.dist-info}/WHEEL +1 -1
- {eodag-3.0.0b3.dist-info → eodag-3.1.0.dist-info}/entry_points.txt +5 -2
- eodag/resources/constraints/climate-dt.json +0 -13
- eodag/resources/constraints/extremes-dt.json +0 -8
- eodag/utils/constraints.py +0 -244
- eodag-3.0.0b3.dist-info/RECORD +0 -110
- {eodag-3.0.0b3.dist-info → eodag-3.1.0.dist-info}/LICENSE +0 -0
- {eodag-3.0.0b3.dist-info → eodag-3.1.0.dist-info}/top_level.txt +0 -0
eodag/plugins/apis/base.py
CHANGED
|
@@ -24,9 +24,10 @@ from eodag.plugins.search.base import Search
|
|
|
24
24
|
class Api(Search, Download):
|
|
25
25
|
"""Plugins API Base plugin
|
|
26
26
|
|
|
27
|
-
An Api plugin
|
|
27
|
+
An Api plugin inherits the methods from Search and Download plugins.
|
|
28
28
|
|
|
29
29
|
There are three methods that it must implement:
|
|
30
|
+
|
|
30
31
|
- ``query``: search for products
|
|
31
32
|
- ``download``: download a single :class:`~eodag.api.product._product.EOProduct`
|
|
32
33
|
- ``download_all``: download multiple products from a :class:`~eodag.api.search_result.SearchResult`
|
|
@@ -35,14 +36,14 @@ class Api(Search, Download):
|
|
|
35
36
|
|
|
36
37
|
- download data in the ``output_dir`` folder defined in the plugin's
|
|
37
38
|
configuration or passed through kwargs
|
|
38
|
-
- extract products from their archive (if relevant) if ``extract`` is set to True
|
|
39
|
-
(True by default)
|
|
39
|
+
- extract products from their archive (if relevant) if ``extract`` is set to ``True``
|
|
40
|
+
(``True`` by default)
|
|
40
41
|
- save a product in an archive/directory (in ``output_dir``) whose name must be
|
|
41
42
|
the product's ``title`` property
|
|
42
43
|
- update the product's ``location`` attribute once its data is downloaded (and
|
|
43
44
|
eventually after it's extracted) to the product's location given as a file URI
|
|
44
45
|
(e.g. 'file:///tmp/product_folder' on Linux or
|
|
45
|
-
'file:///C:/Users/username/AppData/
|
|
46
|
+
'file:///C:/Users/username/AppData/Local/Temp' on Windows)
|
|
46
47
|
- save a *record* file in the directory ``output_dir/.downloaded`` whose name
|
|
47
48
|
is built on the MD5 hash of the product's ``product_type`` and ``properties['id']``
|
|
48
49
|
attributes (``hashlib.md5((product.product_type+"-"+product.properties['id']).encode("utf-8")).hexdigest()``)
|
|
@@ -52,4 +53,9 @@ class Api(Search, Download):
|
|
|
52
53
|
- not try to download a product if its *record* file exists as long as the expected
|
|
53
54
|
product's file/directory. If the *record* file only is found, it must be deleted
|
|
54
55
|
(it certainly indicates that the download didn't complete)
|
|
56
|
+
|
|
57
|
+
:param provider: An EODAG provider name
|
|
58
|
+
:type provider: str
|
|
59
|
+
:param config: An EODAG plugin configuration
|
|
60
|
+
:type config: dict[str, Any]
|
|
55
61
|
"""
|
eodag/plugins/apis/ecmwf.py
CHANGED
|
@@ -20,16 +20,17 @@ from __future__ import annotations
|
|
|
20
20
|
import logging
|
|
21
21
|
import os
|
|
22
22
|
from datetime import datetime, timezone
|
|
23
|
-
from typing import TYPE_CHECKING
|
|
23
|
+
from typing import TYPE_CHECKING, Annotated
|
|
24
24
|
|
|
25
25
|
import geojson
|
|
26
26
|
from ecmwfapi import ECMWFDataServer, ECMWFService
|
|
27
27
|
from ecmwfapi.api import APIException, Connection, get_apikey_values
|
|
28
|
+
from pydantic.fields import FieldInfo
|
|
28
29
|
|
|
29
30
|
from eodag.plugins.apis.base import Api
|
|
30
31
|
from eodag.plugins.search import PreparedSearch
|
|
31
32
|
from eodag.plugins.search.base import Search
|
|
32
|
-
from eodag.plugins.search.build_search_result import
|
|
33
|
+
from eodag.plugins.search.build_search_result import ECMWFSearch, ecmwf_mtd
|
|
33
34
|
from eodag.utils import (
|
|
34
35
|
DEFAULT_DOWNLOAD_TIMEOUT,
|
|
35
36
|
DEFAULT_DOWNLOAD_WAIT,
|
|
@@ -43,13 +44,14 @@ from eodag.utils.exceptions import AuthenticationError, DownloadError
|
|
|
43
44
|
from eodag.utils.logging import get_logging_verbose
|
|
44
45
|
|
|
45
46
|
if TYPE_CHECKING:
|
|
46
|
-
from typing import Any,
|
|
47
|
+
from typing import Any, Optional, Union
|
|
47
48
|
|
|
48
49
|
from requests.auth import AuthBase
|
|
49
50
|
|
|
50
51
|
from eodag.api.product import EOProduct
|
|
51
52
|
from eodag.api.search_result import SearchResult
|
|
52
53
|
from eodag.config import PluginConfig
|
|
54
|
+
from eodag.types import S3SessionKwargs
|
|
53
55
|
from eodag.types.download_args import DownloadConf
|
|
54
56
|
from eodag.utils import DownloadedCallback, ProgressCallback, Unpack
|
|
55
57
|
|
|
@@ -58,7 +60,7 @@ logger = logging.getLogger("eodag.apis.ecmwf")
|
|
|
58
60
|
ECMWF_MARS_KNOWN_FORMATS = {"grib": "grib", "netcdf": "nc"}
|
|
59
61
|
|
|
60
62
|
|
|
61
|
-
class EcmwfApi(Api,
|
|
63
|
+
class EcmwfApi(Api, ECMWFSearch):
|
|
62
64
|
"""A plugin that enables to build download-request and download data on ECMWF MARS.
|
|
63
65
|
|
|
64
66
|
Builds a single ready-to-download :class:`~eodag.api.product._product.EOProduct`
|
|
@@ -68,22 +70,38 @@ class EcmwfApi(Api, BuildPostSearchResult):
|
|
|
68
70
|
is in query), or on MARS Operational Archive (if ``dataset`` parameter is not in
|
|
69
71
|
query).
|
|
70
72
|
|
|
71
|
-
This class inherits from :class:`~eodag.plugins.apis.base.Api` for compatibility
|
|
72
|
-
:class:`~eodag.plugins.
|
|
73
|
-
|
|
74
|
-
|
|
73
|
+
This class inherits from :class:`~eodag.plugins.apis.base.Api` for compatibility and
|
|
74
|
+
:class:`~eodag.plugins.search.build_search_result.ECMWFSearch` for the creation
|
|
75
|
+
of the search result.
|
|
76
|
+
|
|
77
|
+
:param provider: provider name
|
|
78
|
+
:param config: Api plugin configuration:
|
|
79
|
+
|
|
80
|
+
* :attr:`~eodag.config.PluginConfig.type` (``str``) (**mandatory**): EcmwfApi
|
|
81
|
+
* :attr:`~eodag.config.PluginConfig.auth_endpoint` (``str``) (**mandatory**): url of
|
|
82
|
+
the authentication endpoint of the ecmwf api
|
|
83
|
+
* :attr:`~eodag.config.PluginConfig.metadata_mapping` (``dict[str, Union[str, list]]``): how
|
|
84
|
+
parameters should be mapped between the provider and eodag; If a string is given, this is
|
|
85
|
+
the mapping parameter returned by provider -> eodag parameter. If a list with 2 elements
|
|
86
|
+
is given, the first one is the mapping eodag parameter -> provider query parameters
|
|
87
|
+
and the second one the mapping provider result parameter -> eodag parameter
|
|
75
88
|
"""
|
|
76
89
|
|
|
77
90
|
def __init__(self, provider: str, config: PluginConfig) -> None:
|
|
78
91
|
# init self.config.metadata_mapping using Search Base plugin
|
|
92
|
+
config.metadata_mapping = {
|
|
93
|
+
**ecmwf_mtd(),
|
|
94
|
+
**config.metadata_mapping,
|
|
95
|
+
}
|
|
79
96
|
Search.__init__(self, provider, config)
|
|
80
97
|
|
|
81
98
|
# needed by QueryStringSearch.build_query_string / format_free_text_search
|
|
82
99
|
self.config.__dict__.setdefault("free_text_search_operations", {})
|
|
83
100
|
# needed for compatibility
|
|
84
101
|
self.config.__dict__.setdefault("pagination", {"next_page_query_obj": "{{}}"})
|
|
102
|
+
self.config.__dict__.setdefault("api_endpoint", "")
|
|
85
103
|
|
|
86
|
-
def do_search(self, *args: Any, **kwargs: Any) ->
|
|
104
|
+
def do_search(self, *args: Any, **kwargs: Any) -> list[dict[str, Any]]:
|
|
87
105
|
"""Should perform the actual search request."""
|
|
88
106
|
return [{}]
|
|
89
107
|
|
|
@@ -91,16 +109,16 @@ class EcmwfApi(Api, BuildPostSearchResult):
|
|
|
91
109
|
self,
|
|
92
110
|
prep: PreparedSearch = PreparedSearch(),
|
|
93
111
|
**kwargs: Any,
|
|
94
|
-
) ->
|
|
112
|
+
) -> tuple[list[EOProduct], Optional[int]]:
|
|
95
113
|
"""Build ready-to-download SearchResult"""
|
|
96
114
|
|
|
97
115
|
# check productType, dates, geometry, use defaults if not specified
|
|
98
116
|
# productType
|
|
99
117
|
if not kwargs.get("productType"):
|
|
100
118
|
kwargs["productType"] = "%s_%s_%s" % (
|
|
101
|
-
kwargs.get("dataset", "mars"),
|
|
102
|
-
kwargs.get("type", ""),
|
|
103
|
-
kwargs.get("levtype", ""),
|
|
119
|
+
kwargs.get("ecmwf:dataset", "mars"),
|
|
120
|
+
kwargs.get("ecmwf:type", ""),
|
|
121
|
+
kwargs.get("ecmwf:levtype", ""),
|
|
104
122
|
)
|
|
105
123
|
# start date
|
|
106
124
|
if "startTimeFromAscendingNode" not in kwargs:
|
|
@@ -122,9 +140,9 @@ class EcmwfApi(Api, BuildPostSearchResult):
|
|
|
122
140
|
if "geometry" in kwargs:
|
|
123
141
|
kwargs["geometry"] = get_geometry_from_various(geometry=kwargs["geometry"])
|
|
124
142
|
|
|
125
|
-
return
|
|
143
|
+
return ECMWFSearch.query(self, prep, **kwargs)
|
|
126
144
|
|
|
127
|
-
def authenticate(self) ->
|
|
145
|
+
def authenticate(self) -> dict[str, Optional[str]]:
|
|
128
146
|
"""Check credentials and returns information needed for auth
|
|
129
147
|
|
|
130
148
|
:returns: {key, url, email} dictionary
|
|
@@ -133,7 +151,7 @@ class EcmwfApi(Api, BuildPostSearchResult):
|
|
|
133
151
|
# Get credentials from eodag or using ecmwf conf
|
|
134
152
|
email = getattr(self.config, "credentials", {}).get("username", None)
|
|
135
153
|
key = getattr(self.config, "credentials", {}).get("password", None)
|
|
136
|
-
url = getattr(self.config, "
|
|
154
|
+
url = getattr(self.config, "auth_endpoint", None)
|
|
137
155
|
if not all([email, key, url]):
|
|
138
156
|
key, url, email = get_apikey_values()
|
|
139
157
|
|
|
@@ -155,10 +173,10 @@ class EcmwfApi(Api, BuildPostSearchResult):
|
|
|
155
173
|
def download(
|
|
156
174
|
self,
|
|
157
175
|
product: EOProduct,
|
|
158
|
-
auth: Optional[Union[AuthBase,
|
|
176
|
+
auth: Optional[Union[AuthBase, S3SessionKwargs]] = None,
|
|
159
177
|
progress_callback: Optional[ProgressCallback] = None,
|
|
160
|
-
wait:
|
|
161
|
-
timeout:
|
|
178
|
+
wait: float = DEFAULT_DOWNLOAD_WAIT,
|
|
179
|
+
timeout: float = DEFAULT_DOWNLOAD_TIMEOUT,
|
|
162
180
|
**kwargs: Unpack[DownloadConf],
|
|
163
181
|
) -> Optional[str]:
|
|
164
182
|
"""Download data from ECMWF MARS"""
|
|
@@ -244,13 +262,13 @@ class EcmwfApi(Api, BuildPostSearchResult):
|
|
|
244
262
|
def download_all(
|
|
245
263
|
self,
|
|
246
264
|
products: SearchResult,
|
|
247
|
-
auth: Optional[Union[AuthBase,
|
|
265
|
+
auth: Optional[Union[AuthBase, S3SessionKwargs]] = None,
|
|
248
266
|
downloaded_callback: Optional[DownloadedCallback] = None,
|
|
249
267
|
progress_callback: Optional[ProgressCallback] = None,
|
|
250
|
-
wait:
|
|
251
|
-
timeout:
|
|
268
|
+
wait: float = DEFAULT_DOWNLOAD_WAIT,
|
|
269
|
+
timeout: float = DEFAULT_DOWNLOAD_TIMEOUT,
|
|
252
270
|
**kwargs: Unpack[DownloadConf],
|
|
253
|
-
) ->
|
|
271
|
+
) -> list[str]:
|
|
254
272
|
"""
|
|
255
273
|
Download all using parent (base plugin) method
|
|
256
274
|
"""
|
|
@@ -267,3 +285,15 @@ class EcmwfApi(Api, BuildPostSearchResult):
|
|
|
267
285
|
def clear(self) -> None:
|
|
268
286
|
"""Clear search context"""
|
|
269
287
|
pass
|
|
288
|
+
|
|
289
|
+
def discover_queryables(
|
|
290
|
+
self, **kwargs: Any
|
|
291
|
+
) -> Optional[dict[str, Annotated[Any, FieldInfo]]]:
|
|
292
|
+
"""Fetch queryables list from provider using metadata mapping
|
|
293
|
+
|
|
294
|
+
:param kwargs: additional filters for queryables (`productType` and other search
|
|
295
|
+
arguments)
|
|
296
|
+
:returns: fetched queryable parameters dict
|
|
297
|
+
"""
|
|
298
|
+
product_type = kwargs.get("productType", None)
|
|
299
|
+
return self.queryables_from_metadata_mapping(product_type)
|
eodag/plugins/apis/usgs.py
CHANGED
|
@@ -22,7 +22,7 @@ import os
|
|
|
22
22
|
import shutil
|
|
23
23
|
import tarfile
|
|
24
24
|
import zipfile
|
|
25
|
-
from typing import TYPE_CHECKING, Any,
|
|
25
|
+
from typing import TYPE_CHECKING, Any, Optional, Union, cast
|
|
26
26
|
|
|
27
27
|
import requests
|
|
28
28
|
from jsonpath_ng.ext import parse
|
|
@@ -61,6 +61,7 @@ if TYPE_CHECKING:
|
|
|
61
61
|
|
|
62
62
|
from eodag.api.search_result import SearchResult
|
|
63
63
|
from eodag.config import PluginConfig
|
|
64
|
+
from eodag.types import S3SessionKwargs
|
|
64
65
|
from eodag.types.download_args import DownloadConf
|
|
65
66
|
from eodag.utils import DownloadedCallback, Unpack
|
|
66
67
|
|
|
@@ -68,7 +69,30 @@ logger = logging.getLogger("eodag.apis.usgs")
|
|
|
68
69
|
|
|
69
70
|
|
|
70
71
|
class UsgsApi(Api):
|
|
71
|
-
"""A plugin that enables to query and download data on the USGS catalogues
|
|
72
|
+
"""A plugin that enables to query and download data on the USGS catalogues
|
|
73
|
+
|
|
74
|
+
:param provider: provider name
|
|
75
|
+
:param config: Api plugin configuration:
|
|
76
|
+
|
|
77
|
+
* :attr:`~eodag.config.PluginConfig.type` (``str``) (**mandatory**): UsgsApi
|
|
78
|
+
* :attr:`~eodag.config.PluginConfig.pagination` (:class:`~eodag.config.PluginConfig.Pagination`)
|
|
79
|
+
(**mandatory**): object containing parameters for pagination; should contain the attribute
|
|
80
|
+
:attr:`~eodag.config.PluginConfig.Pagination.total_items_nb_key_path`
|
|
81
|
+
which is indicating the key for the number of total items in the provider result
|
|
82
|
+
* :attr:`~eodag.config.PluginConfig.ssl_verify` (``bool``): if the ssl certificates
|
|
83
|
+
should be verified in the download request; default: ``True``
|
|
84
|
+
* :attr:`~eodag.config.PluginConfig.need_auth` (``bool``): if authentication is required
|
|
85
|
+
for search; default: ``False``
|
|
86
|
+
* :attr:`~eodag.config.PluginConfig.extract` (``bool``): if the content of the downloaded
|
|
87
|
+
file should be extracted; default: ``True``
|
|
88
|
+
* :attr:`~eodag.config.PluginConfig.order_enabled` (``bool``): if the product has to
|
|
89
|
+
be ordered to download it; default: ``False``
|
|
90
|
+
* :attr:`~eodag.config.PluginConfig.metadata_mapping` (``dict[str, Union[str, list]]``): how
|
|
91
|
+
parameters should be mapped between the provider and eodag; If a string is given, this is
|
|
92
|
+
the mapping parameter returned by provider -> eodag parameter. If a list with 2 elements
|
|
93
|
+
is given, the first one is the mapping eodag parameter -> provider query parameters
|
|
94
|
+
and the second one the mapping provider result parameter -> eodag parameter
|
|
95
|
+
"""
|
|
72
96
|
|
|
73
97
|
def __init__(self, provider: str, config: PluginConfig) -> None:
|
|
74
98
|
super(UsgsApi, self).__init__(provider, config)
|
|
@@ -76,7 +100,7 @@ class UsgsApi(Api):
|
|
|
76
100
|
# Same method as in base.py, Search.__init__()
|
|
77
101
|
# Prepare the metadata mapping
|
|
78
102
|
# Do a shallow copy, the structure is flat enough for this to be sufficient
|
|
79
|
-
metas:
|
|
103
|
+
metas: dict[str, Any] = DEFAULT_METADATA_MAPPING.copy()
|
|
80
104
|
# Update the defaults with the mapping value. This will add any new key
|
|
81
105
|
# added by the provider mapping that is not in the default metadata.
|
|
82
106
|
metas.update(self.config.metadata_mapping)
|
|
@@ -104,7 +128,7 @@ class UsgsApi(Api):
|
|
|
104
128
|
api.logout()
|
|
105
129
|
continue
|
|
106
130
|
except USGSError as e:
|
|
107
|
-
if i == 0:
|
|
131
|
+
if i == 0 and os.path.isfile(api.TMPFILE):
|
|
108
132
|
# `.usgs` API file key might be obsolete
|
|
109
133
|
# Remove it and try again
|
|
110
134
|
os.remove(api.TMPFILE)
|
|
@@ -115,7 +139,7 @@ class UsgsApi(Api):
|
|
|
115
139
|
self,
|
|
116
140
|
prep: PreparedSearch = PreparedSearch(),
|
|
117
141
|
**kwargs: Any,
|
|
118
|
-
) ->
|
|
142
|
+
) -> tuple[list[EOProduct], Optional[int]]:
|
|
119
143
|
"""Search for data on USGS catalogues"""
|
|
120
144
|
page = prep.page if prep.page is not None else DEFAULT_PAGE
|
|
121
145
|
items_per_page = (
|
|
@@ -141,7 +165,7 @@ class UsgsApi(Api):
|
|
|
141
165
|
start_date = kwargs.pop("startTimeFromAscendingNode", None)
|
|
142
166
|
end_date = kwargs.pop("completionTimeFromAscendingNode", None)
|
|
143
167
|
geom = kwargs.pop("geometry", None)
|
|
144
|
-
footprint:
|
|
168
|
+
footprint: dict[str, str] = {}
|
|
145
169
|
if hasattr(geom, "bounds"):
|
|
146
170
|
(
|
|
147
171
|
footprint["lonmin"],
|
|
@@ -152,7 +176,7 @@ class UsgsApi(Api):
|
|
|
152
176
|
else:
|
|
153
177
|
footprint = geom
|
|
154
178
|
|
|
155
|
-
final:
|
|
179
|
+
final: list[EOProduct] = []
|
|
156
180
|
if footprint and len(footprint.keys()) == 4: # a rectangle (or bbox)
|
|
157
181
|
lower_left = {
|
|
158
182
|
"longitude": footprint["lonmin"],
|
|
@@ -256,7 +280,7 @@ class UsgsApi(Api):
|
|
|
256
280
|
f"Product type {usgs_dataset} may not exist on USGS EE catalog"
|
|
257
281
|
)
|
|
258
282
|
api.logout()
|
|
259
|
-
raise RequestError(e)
|
|
283
|
+
raise RequestError.from_error(e) from e
|
|
260
284
|
|
|
261
285
|
api.logout()
|
|
262
286
|
|
|
@@ -272,10 +296,10 @@ class UsgsApi(Api):
|
|
|
272
296
|
def download(
|
|
273
297
|
self,
|
|
274
298
|
product: EOProduct,
|
|
275
|
-
auth: Optional[Union[AuthBase,
|
|
299
|
+
auth: Optional[Union[AuthBase, S3SessionKwargs]] = None,
|
|
276
300
|
progress_callback: Optional[ProgressCallback] = None,
|
|
277
|
-
wait:
|
|
278
|
-
timeout:
|
|
301
|
+
wait: float = DEFAULT_DOWNLOAD_WAIT,
|
|
302
|
+
timeout: float = DEFAULT_DOWNLOAD_TIMEOUT,
|
|
279
303
|
**kwargs: Unpack[DownloadConf],
|
|
280
304
|
) -> Optional[str]:
|
|
281
305
|
"""Download data from USGS catalogues"""
|
|
@@ -317,7 +341,7 @@ class UsgsApi(Api):
|
|
|
317
341
|
product.properties["productId"],
|
|
318
342
|
)
|
|
319
343
|
|
|
320
|
-
req_urls:
|
|
344
|
+
req_urls: list[str] = []
|
|
321
345
|
try:
|
|
322
346
|
if len(download_request_results["data"]["preparingDownloads"]) > 0:
|
|
323
347
|
req_urls.extend(
|
|
@@ -352,7 +376,7 @@ class UsgsApi(Api):
|
|
|
352
376
|
logger.debug(f"Downloading {req_url}")
|
|
353
377
|
ssl_verify = getattr(self.config, "ssl_verify", True)
|
|
354
378
|
|
|
355
|
-
@self.
|
|
379
|
+
@self._order_download_retry(product, wait, timeout)
|
|
356
380
|
def download_request(
|
|
357
381
|
product: EOProduct,
|
|
358
382
|
fs_path: str,
|
|
@@ -441,13 +465,13 @@ class UsgsApi(Api):
|
|
|
441
465
|
def download_all(
|
|
442
466
|
self,
|
|
443
467
|
products: SearchResult,
|
|
444
|
-
auth: Optional[Union[AuthBase,
|
|
468
|
+
auth: Optional[Union[AuthBase, S3SessionKwargs]] = None,
|
|
445
469
|
downloaded_callback: Optional[DownloadedCallback] = None,
|
|
446
470
|
progress_callback: Optional[ProgressCallback] = None,
|
|
447
|
-
wait:
|
|
448
|
-
timeout:
|
|
471
|
+
wait: float = DEFAULT_DOWNLOAD_WAIT,
|
|
472
|
+
timeout: float = DEFAULT_DOWNLOAD_TIMEOUT,
|
|
449
473
|
**kwargs: Unpack[DownloadConf],
|
|
450
|
-
) ->
|
|
474
|
+
) -> list[str]:
|
|
451
475
|
"""
|
|
452
476
|
Download all using parent (base plugin) method
|
|
453
477
|
"""
|
|
@@ -17,9 +17,10 @@
|
|
|
17
17
|
# limitations under the License.
|
|
18
18
|
from __future__ import annotations
|
|
19
19
|
|
|
20
|
-
from typing import TYPE_CHECKING,
|
|
20
|
+
from typing import TYPE_CHECKING, Optional, cast
|
|
21
21
|
|
|
22
22
|
from eodag.plugins.authentication.base import Authentication
|
|
23
|
+
from eodag.types import S3SessionKwargs
|
|
23
24
|
|
|
24
25
|
if TYPE_CHECKING:
|
|
25
26
|
from mypy_boto3_s3.client import S3Client
|
|
@@ -30,26 +31,35 @@ if TYPE_CHECKING:
|
|
|
30
31
|
class AwsAuth(Authentication):
|
|
31
32
|
"""AWS authentication plugin
|
|
32
33
|
|
|
33
|
-
Authentication will use the first valid method within the following ones
|
|
34
|
+
Authentication will use the first valid method within the following ones depending on which
|
|
35
|
+
parameters are available in the configuration:
|
|
34
36
|
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
37
|
+
* auth anonymously using no-sign-request
|
|
38
|
+
* auth using ``aws_profile``
|
|
39
|
+
* auth using ``aws_access_key_id`` and ``aws_secret_access_key``
|
|
38
40
|
(optionally ``aws_session_token``)
|
|
39
|
-
|
|
41
|
+
* auth using current environment (AWS environment variables and/or ``~/aws/*``),
|
|
40
42
|
will be skipped if AWS credentials are filled in eodag conf
|
|
43
|
+
|
|
44
|
+
:param provider: provider name
|
|
45
|
+
:param config: Authentication plugin configuration:
|
|
46
|
+
|
|
47
|
+
* :attr:`~eodag.config.PluginConfig.type` (``str``) (**mandatory**): AwsAuth
|
|
48
|
+
* :attr:`~eodag.config.PluginConfig.auth_error_code` (``int``) (mandatory for ``creodias_s3``):
|
|
49
|
+
which error code is returned in case of an authentication error
|
|
50
|
+
|
|
41
51
|
"""
|
|
42
52
|
|
|
43
53
|
s3_client: S3Client
|
|
44
54
|
|
|
45
55
|
def __init__(self, provider: str, config: PluginConfig) -> None:
|
|
46
56
|
super(AwsAuth, self).__init__(provider, config)
|
|
47
|
-
self.aws_access_key_id = None
|
|
48
|
-
self.aws_secret_access_key = None
|
|
49
|
-
self.aws_session_token = None
|
|
50
|
-
self.profile_name = None
|
|
57
|
+
self.aws_access_key_id: Optional[str] = None
|
|
58
|
+
self.aws_secret_access_key: Optional[str] = None
|
|
59
|
+
self.aws_session_token: Optional[str] = None
|
|
60
|
+
self.profile_name: Optional[str] = None
|
|
51
61
|
|
|
52
|
-
def authenticate(self) ->
|
|
62
|
+
def authenticate(self) -> S3SessionKwargs:
|
|
53
63
|
"""Authenticate
|
|
54
64
|
|
|
55
65
|
:returns: dict containing AWS/boto3 non-empty credentials
|
|
@@ -66,10 +76,12 @@ class AwsAuth(Authentication):
|
|
|
66
76
|
)
|
|
67
77
|
self.profile_name = credentials.get("aws_profile", self.profile_name)
|
|
68
78
|
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
|
|
79
|
+
auth_dict = cast(
|
|
80
|
+
S3SessionKwargs,
|
|
81
|
+
{
|
|
82
|
+
k: getattr(self, k)
|
|
83
|
+
for k in S3SessionKwargs.__annotations__
|
|
84
|
+
if getattr(self, k, None)
|
|
85
|
+
},
|
|
86
|
+
)
|
|
87
|
+
return auth_dict
|
|
@@ -17,7 +17,7 @@
|
|
|
17
17
|
# limitations under the License.
|
|
18
18
|
from __future__ import annotations
|
|
19
19
|
|
|
20
|
-
from typing import TYPE_CHECKING,
|
|
20
|
+
from typing import TYPE_CHECKING, Union
|
|
21
21
|
|
|
22
22
|
from eodag.plugins.base import PluginTopic
|
|
23
23
|
from eodag.utils.exceptions import MisconfiguredError
|
|
@@ -25,11 +25,22 @@ from eodag.utils.exceptions import MisconfiguredError
|
|
|
25
25
|
if TYPE_CHECKING:
|
|
26
26
|
from requests.auth import AuthBase
|
|
27
27
|
|
|
28
|
+
from eodag.types import S3SessionKwargs
|
|
29
|
+
|
|
28
30
|
|
|
29
31
|
class Authentication(PluginTopic):
|
|
30
|
-
"""Plugins authentication Base plugin
|
|
32
|
+
"""Plugins authentication Base plugin
|
|
33
|
+
|
|
34
|
+
:param provider: provider name
|
|
35
|
+
:param config: Authentication plugin configuration:
|
|
36
|
+
|
|
37
|
+
* :attr:`~eodag.config.PluginConfig.matching_url` (``str``): URL pattern to match with search plugin endpoint or
|
|
38
|
+
download link
|
|
39
|
+
* :attr:`~eodag.config.PluginConfig.matching_conf` (``dict[str, Any]``): Part of the search or download plugin
|
|
40
|
+
configuration that needs authentication and helps identifying it
|
|
41
|
+
"""
|
|
31
42
|
|
|
32
|
-
def authenticate(self) -> Union[AuthBase,
|
|
43
|
+
def authenticate(self) -> Union[AuthBase, S3SessionKwargs]:
|
|
33
44
|
"""Authenticate"""
|
|
34
45
|
raise NotImplementedError
|
|
35
46
|
|
|
@@ -29,7 +29,18 @@ if TYPE_CHECKING:
|
|
|
29
29
|
|
|
30
30
|
|
|
31
31
|
class GenericAuth(Authentication):
|
|
32
|
-
"""GenericAuth authentication plugin
|
|
32
|
+
"""GenericAuth authentication plugin (authentication using ``username`` and ``password``)
|
|
33
|
+
|
|
34
|
+
The mandatory parameters that have to be added in the eodag config are username and password.
|
|
35
|
+
|
|
36
|
+
:param provider: provider name
|
|
37
|
+
:param config: Authentication plugin configuration:
|
|
38
|
+
|
|
39
|
+
* :attr:`~eodag.config.PluginConfig.type` (``str``) (**mandatory**): GenericAuth
|
|
40
|
+
* :attr:`~eodag.config.PluginConfig.method` (``str``): specifies if digest authentication
|
|
41
|
+
(``digest``) or basic authentication (``basic``) should be used; default: ``basic``
|
|
42
|
+
|
|
43
|
+
"""
|
|
33
44
|
|
|
34
45
|
def authenticate(self) -> AuthBase:
|
|
35
46
|
"""Authenticate"""
|
|
@@ -48,6 +59,6 @@ class GenericAuth(Authentication):
|
|
|
48
59
|
)
|
|
49
60
|
else:
|
|
50
61
|
raise MisconfiguredError(
|
|
51
|
-
f"Cannot authenticate with {self.provider}
|
|
52
|
-
f"
|
|
62
|
+
f"Cannot authenticate with {self.provider}",
|
|
63
|
+
f"Method {method} is not supported; it must be one of 'digest' or 'basic'.",
|
|
53
64
|
)
|
|
@@ -17,7 +17,7 @@
|
|
|
17
17
|
# limitations under the License.
|
|
18
18
|
from __future__ import annotations
|
|
19
19
|
|
|
20
|
-
from typing import TYPE_CHECKING
|
|
20
|
+
from typing import TYPE_CHECKING
|
|
21
21
|
|
|
22
22
|
from requests.auth import AuthBase
|
|
23
23
|
|
|
@@ -34,7 +34,14 @@ class HTTPHeaderAuth(Authentication):
|
|
|
34
34
|
This plugin enables implementation of custom HTTP authentication scheme (other than Basic, Digest, Token
|
|
35
35
|
negotiation et al.) using HTTP headers.
|
|
36
36
|
|
|
37
|
-
|
|
37
|
+
:param provider: provider name
|
|
38
|
+
:param config: Authentication plugin configuration:
|
|
39
|
+
|
|
40
|
+
* :attr:`~eodag.config.PluginConfig.type` (``str``) (**mandatory**): HTTPHeaderAuth
|
|
41
|
+
* :attr:`~eodag.config.PluginConfig.headers` (``dict[str, str]``): dictionary containing
|
|
42
|
+
all keys/value pairs that should be added to the headers
|
|
43
|
+
|
|
44
|
+
Below an example for the configuration in the providers config file is shown::
|
|
38
45
|
|
|
39
46
|
provider:
|
|
40
47
|
...
|
|
@@ -47,9 +54,9 @@ class HTTPHeaderAuth(Authentication):
|
|
|
47
54
|
...
|
|
48
55
|
...
|
|
49
56
|
|
|
50
|
-
As you can see in the sample above, the maintainer of
|
|
51
|
-
authentication process as-is, by giving their names (e.g.
|
|
52
|
-
|
|
57
|
+
As you can see in the sample above, the maintainer of ``provider`` define the headers that will be used in the
|
|
58
|
+
authentication process as-is, by giving their names (e.g. ``Authorization``) and their value (e.g
|
|
59
|
+
``"Something {userinput}"``) as regular Python string templates that enable passing in the user input necessary to
|
|
53
60
|
compute its identity. The user input awaited in the header value string must be present in the user config file.
|
|
54
61
|
In the sample above, the plugin await for user credentials to be specified as::
|
|
55
62
|
|
|
@@ -72,6 +79,7 @@ class HTTPHeaderAuth(Authentication):
|
|
|
72
79
|
X-Another-Special-Header: "YYY"
|
|
73
80
|
...
|
|
74
81
|
...
|
|
82
|
+
|
|
75
83
|
"""
|
|
76
84
|
|
|
77
85
|
def authenticate(self) -> HeaderAuth:
|
|
@@ -98,7 +106,7 @@ class HTTPHeaderAuth(Authentication):
|
|
|
98
106
|
class HeaderAuth(AuthBase):
|
|
99
107
|
"""HeaderAuth custom authentication class to be used with requests module"""
|
|
100
108
|
|
|
101
|
-
def __init__(self, authentication_headers:
|
|
109
|
+
def __init__(self, authentication_headers: dict[str, str]) -> None:
|
|
102
110
|
self.auth_headers = authentication_headers
|
|
103
111
|
|
|
104
112
|
def __call__(self, request: PreparedRequest) -> PreparedRequest:
|