eodag 3.1.0b1__py3-none-any.whl → 3.2.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- eodag/api/core.py +69 -63
- eodag/api/product/_assets.py +49 -13
- eodag/api/product/_product.py +41 -30
- eodag/api/product/drivers/__init__.py +81 -4
- eodag/api/product/drivers/base.py +65 -4
- eodag/api/product/drivers/generic.py +65 -0
- eodag/api/product/drivers/sentinel1.py +97 -0
- eodag/api/product/drivers/sentinel2.py +95 -0
- eodag/api/product/metadata_mapping.py +85 -79
- eodag/api/search_result.py +13 -23
- eodag/cli.py +4 -4
- eodag/config.py +77 -80
- eodag/plugins/apis/base.py +1 -1
- eodag/plugins/apis/ecmwf.py +12 -15
- eodag/plugins/apis/usgs.py +12 -11
- eodag/plugins/authentication/aws_auth.py +16 -13
- eodag/plugins/authentication/base.py +5 -3
- eodag/plugins/authentication/header.py +3 -3
- eodag/plugins/authentication/keycloak.py +4 -4
- eodag/plugins/authentication/oauth.py +7 -3
- eodag/plugins/authentication/openid_connect.py +20 -14
- eodag/plugins/authentication/sas_auth.py +4 -4
- eodag/plugins/authentication/token.py +7 -7
- eodag/plugins/authentication/token_exchange.py +1 -1
- eodag/plugins/base.py +4 -4
- eodag/plugins/crunch/base.py +4 -4
- eodag/plugins/crunch/filter_date.py +4 -4
- eodag/plugins/crunch/filter_latest_intersect.py +6 -6
- eodag/plugins/crunch/filter_latest_tpl_name.py +7 -7
- eodag/plugins/crunch/filter_overlap.py +4 -4
- eodag/plugins/crunch/filter_property.py +4 -4
- eodag/plugins/download/aws.py +137 -77
- eodag/plugins/download/base.py +8 -17
- eodag/plugins/download/creodias_s3.py +2 -2
- eodag/plugins/download/http.py +30 -32
- eodag/plugins/download/s3rest.py +5 -4
- eodag/plugins/manager.py +10 -20
- eodag/plugins/search/__init__.py +6 -5
- eodag/plugins/search/base.py +38 -42
- eodag/plugins/search/build_search_result.py +286 -336
- eodag/plugins/search/cop_marine.py +22 -12
- eodag/plugins/search/creodias_s3.py +8 -78
- eodag/plugins/search/csw.py +11 -11
- eodag/plugins/search/data_request_search.py +19 -18
- eodag/plugins/search/qssearch.py +84 -151
- eodag/plugins/search/stac_list_assets.py +85 -0
- eodag/plugins/search/static_stac_search.py +4 -4
- eodag/resources/ext_product_types.json +1 -1
- eodag/resources/product_types.yml +848 -398
- eodag/resources/providers.yml +1038 -1115
- eodag/resources/stac_api.yml +2 -2
- eodag/resources/user_conf_template.yml +10 -9
- eodag/rest/cache.py +2 -2
- eodag/rest/config.py +3 -3
- eodag/rest/core.py +24 -24
- eodag/rest/errors.py +5 -5
- eodag/rest/server.py +3 -11
- eodag/rest/stac.py +41 -38
- eodag/rest/types/collections_search.py +3 -3
- eodag/rest/types/eodag_search.py +23 -23
- eodag/rest/types/queryables.py +40 -28
- eodag/rest/types/stac_search.py +15 -25
- eodag/rest/utils/__init__.py +11 -21
- eodag/rest/utils/cql_evaluate.py +6 -6
- eodag/rest/utils/rfc3339.py +2 -2
- eodag/types/__init__.py +97 -29
- eodag/types/bbox.py +2 -2
- eodag/types/download_args.py +2 -2
- eodag/types/queryables.py +5 -2
- eodag/types/search_args.py +4 -4
- eodag/types/whoosh.py +1 -3
- eodag/utils/__init__.py +82 -41
- eodag/utils/exceptions.py +2 -2
- eodag/utils/import_system.py +2 -2
- eodag/utils/requests.py +2 -2
- eodag/utils/rest.py +2 -2
- eodag/utils/s3.py +231 -0
- eodag/utils/stac_reader.py +10 -10
- {eodag-3.1.0b1.dist-info → eodag-3.2.0.dist-info}/METADATA +12 -10
- eodag-3.2.0.dist-info/RECORD +113 -0
- {eodag-3.1.0b1.dist-info → eodag-3.2.0.dist-info}/WHEEL +1 -1
- {eodag-3.1.0b1.dist-info → eodag-3.2.0.dist-info}/entry_points.txt +1 -0
- eodag-3.1.0b1.dist-info/RECORD +0 -108
- {eodag-3.1.0b1.dist-info → eodag-3.2.0.dist-info/licenses}/LICENSE +0 -0
- {eodag-3.1.0b1.dist-info → eodag-3.2.0.dist-info}/top_level.txt +0 -0
|
@@ -15,7 +15,7 @@
|
|
|
15
15
|
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
16
16
|
# See the License for the specific language governing permissions and
|
|
17
17
|
# limitations under the License.
|
|
18
|
-
from typing import
|
|
18
|
+
from typing import Optional
|
|
19
19
|
|
|
20
20
|
import boto3
|
|
21
21
|
from botocore.exceptions import ClientError
|
|
@@ -73,7 +73,7 @@ class CreodiasS3Download(AwsDownload):
|
|
|
73
73
|
product: EOProduct,
|
|
74
74
|
asset_filter: Optional[str] = None,
|
|
75
75
|
ignore_assets: Optional[bool] = False,
|
|
76
|
-
) ->
|
|
76
|
+
) -> list[tuple[str, Optional[str]]]:
|
|
77
77
|
"""
|
|
78
78
|
Retrieves the bucket names and path prefixes for the assets
|
|
79
79
|
|
eodag/plugins/download/http.py
CHANGED
|
@@ -28,17 +28,7 @@ from email.message import Message
|
|
|
28
28
|
from itertools import chain
|
|
29
29
|
from json import JSONDecodeError
|
|
30
30
|
from pathlib import Path
|
|
31
|
-
from typing import
|
|
32
|
-
TYPE_CHECKING,
|
|
33
|
-
Any,
|
|
34
|
-
Dict,
|
|
35
|
-
Iterator,
|
|
36
|
-
List,
|
|
37
|
-
Optional,
|
|
38
|
-
TypedDict,
|
|
39
|
-
Union,
|
|
40
|
-
cast,
|
|
41
|
-
)
|
|
31
|
+
from typing import TYPE_CHECKING, Any, Iterator, Optional, TypedDict, Union, cast
|
|
42
32
|
from urllib.parse import parse_qs, urlparse
|
|
43
33
|
|
|
44
34
|
import geojson
|
|
@@ -72,6 +62,7 @@ from eodag.utils import (
|
|
|
72
62
|
guess_file_type,
|
|
73
63
|
parse_header,
|
|
74
64
|
path_to_uri,
|
|
65
|
+
rename_with_version,
|
|
75
66
|
sanitize,
|
|
76
67
|
string_to_jsonpath,
|
|
77
68
|
uri_to_path,
|
|
@@ -91,6 +82,7 @@ if TYPE_CHECKING:
|
|
|
91
82
|
from eodag.api.product import Asset, EOProduct # type: ignore
|
|
92
83
|
from eodag.api.search_result import SearchResult
|
|
93
84
|
from eodag.config import PluginConfig
|
|
85
|
+
from eodag.types import S3SessionKwargs
|
|
94
86
|
from eodag.types.download_args import DownloadConf
|
|
95
87
|
from eodag.utils import DownloadedCallback, Unpack
|
|
96
88
|
|
|
@@ -111,7 +103,7 @@ class HTTPDownload(Download):
|
|
|
111
103
|
extracted; default: ``True``
|
|
112
104
|
* :attr:`~eodag.config.PluginConfig.auth_error_code` (``int``): which error code is returned in case of an
|
|
113
105
|
authentication error
|
|
114
|
-
* :attr:`~eodag.config.PluginConfig.dl_url_params` (``
|
|
106
|
+
* :attr:`~eodag.config.PluginConfig.dl_url_params` (``dict[str, Any]``): parameters to be
|
|
115
107
|
added to the query params of the request
|
|
116
108
|
* :attr:`~eodag.config.PluginConfig.archive_depth` (``int``): level in extracted path tree where to find data;
|
|
117
109
|
default: ``1``
|
|
@@ -130,7 +122,7 @@ class HTTPDownload(Download):
|
|
|
130
122
|
the search plugin used for the provider; default: ``False``
|
|
131
123
|
* :attr:`~eodag.config.PluginConfig.order_method` (``str``): HTTP request method for the order request (``GET``
|
|
132
124
|
or ``POST``); default: ``GET``
|
|
133
|
-
* :attr:`~eodag.config.PluginConfig.order_headers` (``[
|
|
125
|
+
* :attr:`~eodag.config.PluginConfig.order_headers` (``[dict[str, str]]``): headers to be added to the order
|
|
134
126
|
request
|
|
135
127
|
* :attr:`~eodag.config.PluginConfig.order_on_response` (:class:`~eodag.config.PluginConfig.OrderOnResponse`):
|
|
136
128
|
a typed dictionary containing the key ``metadata_mapping`` which can be used to add new product properties
|
|
@@ -138,7 +130,7 @@ class HTTPDownload(Download):
|
|
|
138
130
|
* :attr:`~eodag.config.PluginConfig.order_status` (:class:`~eodag.config.PluginConfig.OrderStatus`):
|
|
139
131
|
configuration to handle the order status; contains information which method to use, how the response data is
|
|
140
132
|
interpreted, which status corresponds to success, ordered and error and what should be done on success.
|
|
141
|
-
* :attr:`~eodag.config.PluginConfig.products` (``
|
|
133
|
+
* :attr:`~eodag.config.PluginConfig.products` (``dict[str, dict[str, Any]``): product type specific config; the
|
|
142
134
|
keys are the product types, the values are dictionaries which can contain the key
|
|
143
135
|
:attr:`~eodag.config.PluginConfig.extract` to overwrite the provider config for a specific product type
|
|
144
136
|
|
|
@@ -152,7 +144,7 @@ class HTTPDownload(Download):
|
|
|
152
144
|
product: EOProduct,
|
|
153
145
|
auth: Optional[AuthBase] = None,
|
|
154
146
|
**kwargs: Unpack[DownloadConf],
|
|
155
|
-
) -> Optional[
|
|
147
|
+
) -> Optional[dict[str, Any]]:
|
|
156
148
|
"""Send product order request.
|
|
157
149
|
|
|
158
150
|
It will be executed once before the download retry loop, if the product is OFFLINE
|
|
@@ -184,7 +176,7 @@ class HTTPDownload(Download):
|
|
|
184
176
|
ssl_verify = getattr(self.config, "ssl_verify", True)
|
|
185
177
|
timeout = getattr(self.config, "timeout", HTTP_REQ_TIMEOUT)
|
|
186
178
|
OrderKwargs = TypedDict(
|
|
187
|
-
"OrderKwargs", {"json":
|
|
179
|
+
"OrderKwargs", {"json": dict[str, Union[Any, list[str]]]}, total=False
|
|
188
180
|
)
|
|
189
181
|
order_kwargs: OrderKwargs = {}
|
|
190
182
|
if order_method == "POST":
|
|
@@ -236,7 +228,7 @@ class HTTPDownload(Download):
|
|
|
236
228
|
|
|
237
229
|
def order_response_process(
|
|
238
230
|
self, response: Response, product: EOProduct
|
|
239
|
-
) -> Optional[
|
|
231
|
+
) -> Optional[dict[str, Any]]:
|
|
240
232
|
"""Process order response
|
|
241
233
|
|
|
242
234
|
:param response: The order response
|
|
@@ -300,7 +292,7 @@ class HTTPDownload(Download):
|
|
|
300
292
|
def _request(
|
|
301
293
|
url: str,
|
|
302
294
|
method: str = "GET",
|
|
303
|
-
headers: Optional[
|
|
295
|
+
headers: Optional[dict[str, Any]] = None,
|
|
304
296
|
json: Optional[Any] = None,
|
|
305
297
|
timeout: int = HTTP_REQ_TIMEOUT,
|
|
306
298
|
) -> Response:
|
|
@@ -336,7 +328,7 @@ class HTTPDownload(Download):
|
|
|
336
328
|
except requests.exceptions.Timeout as exc:
|
|
337
329
|
raise TimeOutError(exc, timeout=timeout) from exc
|
|
338
330
|
|
|
339
|
-
status_request:
|
|
331
|
+
status_request: dict[str, Any] = status_config.get("request", {})
|
|
340
332
|
status_request_method = str(status_request.get("method", "GET")).upper()
|
|
341
333
|
|
|
342
334
|
if status_request_method == "POST":
|
|
@@ -353,8 +345,8 @@ class HTTPDownload(Download):
|
|
|
353
345
|
|
|
354
346
|
# check header for success before full status request
|
|
355
347
|
skip_parsing_status_response = False
|
|
356
|
-
status_dict:
|
|
357
|
-
config_on_success:
|
|
348
|
+
status_dict: dict[str, Any] = {}
|
|
349
|
+
config_on_success: dict[str, Any] = status_config.get("on_success", {})
|
|
358
350
|
on_success_mm = config_on_success.get("metadata_mapping", {})
|
|
359
351
|
|
|
360
352
|
status_response_content_needed = (
|
|
@@ -438,13 +430,13 @@ class HTTPDownload(Download):
|
|
|
438
430
|
product.properties["orderStatus"] = status_dict.get("status")
|
|
439
431
|
|
|
440
432
|
# handle status error
|
|
441
|
-
errors:
|
|
433
|
+
errors: dict[str, Any] = status_config.get("error", {})
|
|
442
434
|
if errors and errors.items() <= status_dict.items():
|
|
443
435
|
raise DownloadError(
|
|
444
436
|
f"Provider {product.provider} returned: {status_dict.get('error_message', status_message)}"
|
|
445
437
|
)
|
|
446
438
|
|
|
447
|
-
success_status:
|
|
439
|
+
success_status: dict[str, Any] = status_config.get("success", {}).get("status")
|
|
448
440
|
# if not success
|
|
449
441
|
if (success_status and success_status != status_dict.get("status")) or (
|
|
450
442
|
success_code and success_code != response.status_code
|
|
@@ -562,7 +554,7 @@ class HTTPDownload(Download):
|
|
|
562
554
|
def download(
|
|
563
555
|
self,
|
|
564
556
|
product: EOProduct,
|
|
565
|
-
auth: Optional[Union[AuthBase,
|
|
557
|
+
auth: Optional[Union[AuthBase, S3SessionKwargs]] = None,
|
|
566
558
|
progress_callback: Optional[ProgressCallback] = None,
|
|
567
559
|
wait: float = DEFAULT_DOWNLOAD_WAIT,
|
|
568
560
|
timeout: float = DEFAULT_DOWNLOAD_TIMEOUT,
|
|
@@ -666,6 +658,8 @@ class HTTPDownload(Download):
|
|
|
666
658
|
os.path.dirname(path),
|
|
667
659
|
sanitize(product.properties["title"]),
|
|
668
660
|
)
|
|
661
|
+
if os.path.isfile(new_fs_path):
|
|
662
|
+
rename_with_version(new_fs_path)
|
|
669
663
|
if not os.path.isdir(new_fs_path):
|
|
670
664
|
os.makedirs(new_fs_path)
|
|
671
665
|
shutil.move(path, new_fs_path)
|
|
@@ -719,7 +713,7 @@ class HTTPDownload(Download):
|
|
|
719
713
|
def _stream_download_dict(
|
|
720
714
|
self,
|
|
721
715
|
product: EOProduct,
|
|
722
|
-
auth: Optional[Union[AuthBase,
|
|
716
|
+
auth: Optional[Union[AuthBase, S3SessionKwargs]] = None,
|
|
723
717
|
progress_callback: Optional[ProgressCallback] = None,
|
|
724
718
|
wait: float = DEFAULT_DOWNLOAD_WAIT,
|
|
725
719
|
timeout: float = DEFAULT_DOWNLOAD_TIMEOUT,
|
|
@@ -778,13 +772,17 @@ class HTTPDownload(Download):
|
|
|
778
772
|
)
|
|
779
773
|
|
|
780
774
|
else:
|
|
775
|
+
# get first chunk to check if it does not contain an error (if it does, that error will be raised)
|
|
776
|
+
first_chunks_tuple = next(chunks_tuples)
|
|
781
777
|
outputs_filename = (
|
|
782
778
|
sanitize(product.properties["title"])
|
|
783
779
|
if "title" in product.properties
|
|
784
780
|
else sanitize(product.properties.get("id", "download"))
|
|
785
781
|
)
|
|
786
782
|
return StreamResponse(
|
|
787
|
-
content=stream_zip(
|
|
783
|
+
content=stream_zip(
|
|
784
|
+
chain(iter([first_chunks_tuple]), chunks_tuples)
|
|
785
|
+
),
|
|
788
786
|
media_type="application/zip",
|
|
789
787
|
headers={
|
|
790
788
|
"content-disposition": f"attachment; filename={outputs_filename}.zip",
|
|
@@ -888,7 +886,7 @@ class HTTPDownload(Download):
|
|
|
888
886
|
def order(
|
|
889
887
|
self,
|
|
890
888
|
product: EOProduct,
|
|
891
|
-
auth: Optional[Union[AuthBase,
|
|
889
|
+
auth: Optional[Union[AuthBase, S3SessionKwargs]] = None,
|
|
892
890
|
wait: float = DEFAULT_DOWNLOAD_WAIT,
|
|
893
891
|
timeout: float = DEFAULT_DOWNLOAD_TIMEOUT,
|
|
894
892
|
) -> None:
|
|
@@ -951,7 +949,7 @@ class HTTPDownload(Download):
|
|
|
951
949
|
if not query_dict and parts.query:
|
|
952
950
|
query_dict = geojson.loads(parts.query)
|
|
953
951
|
req_url = parts._replace(query="").geturl()
|
|
954
|
-
req_kwargs:
|
|
952
|
+
req_kwargs: dict[str, Any] = {"json": query_dict} if query_dict else {}
|
|
955
953
|
else:
|
|
956
954
|
req_url = url
|
|
957
955
|
req_kwargs = {}
|
|
@@ -1019,7 +1017,7 @@ class HTTPDownload(Download):
|
|
|
1019
1017
|
product: EOProduct,
|
|
1020
1018
|
auth: Optional[AuthBase] = None,
|
|
1021
1019
|
progress_callback: Optional[ProgressCallback] = None,
|
|
1022
|
-
assets_values:
|
|
1020
|
+
assets_values: list[Asset] = [],
|
|
1023
1021
|
**kwargs: Unpack[DownloadConf],
|
|
1024
1022
|
) -> Iterator[Any]:
|
|
1025
1023
|
if progress_callback is None:
|
|
@@ -1289,9 +1287,9 @@ class HTTPDownload(Download):
|
|
|
1289
1287
|
|
|
1290
1288
|
def _get_asset_sizes(
|
|
1291
1289
|
self,
|
|
1292
|
-
assets_values:
|
|
1290
|
+
assets_values: list[Asset],
|
|
1293
1291
|
auth: Optional[AuthBase],
|
|
1294
|
-
params: Optional[
|
|
1292
|
+
params: Optional[dict[str, str]],
|
|
1295
1293
|
zipped: bool = False,
|
|
1296
1294
|
) -> int:
|
|
1297
1295
|
total_size = 0
|
|
@@ -1364,7 +1362,7 @@ class HTTPDownload(Download):
|
|
|
1364
1362
|
def download_all(
|
|
1365
1363
|
self,
|
|
1366
1364
|
products: SearchResult,
|
|
1367
|
-
auth: Optional[Union[AuthBase,
|
|
1365
|
+
auth: Optional[Union[AuthBase, S3SessionKwargs]] = None,
|
|
1368
1366
|
downloaded_callback: Optional[DownloadedCallback] = None,
|
|
1369
1367
|
progress_callback: Optional[ProgressCallback] = None,
|
|
1370
1368
|
wait: float = DEFAULT_DOWNLOAD_WAIT,
|
eodag/plugins/download/s3rest.py
CHANGED
|
@@ -20,7 +20,7 @@ from __future__ import annotations
|
|
|
20
20
|
import logging
|
|
21
21
|
import os
|
|
22
22
|
import os.path
|
|
23
|
-
from typing import TYPE_CHECKING,
|
|
23
|
+
from typing import TYPE_CHECKING, Optional, Union
|
|
24
24
|
from xml.dom import minidom
|
|
25
25
|
from xml.parsers.expat import ExpatError
|
|
26
26
|
|
|
@@ -54,6 +54,7 @@ from eodag.utils.exceptions import (
|
|
|
54
54
|
if TYPE_CHECKING:
|
|
55
55
|
from eodag.api.product import EOProduct
|
|
56
56
|
from eodag.config import PluginConfig
|
|
57
|
+
from eodag.types import S3SessionKwargs
|
|
57
58
|
from eodag.types.download_args import DownloadConf
|
|
58
59
|
from eodag.utils import Unpack
|
|
59
60
|
|
|
@@ -78,7 +79,7 @@ class S3RestDownload(Download):
|
|
|
78
79
|
* :attr:`~eodag.config.PluginConfig.order_enabled` (``bool``): whether order is enabled
|
|
79
80
|
or not if product is `OFFLINE`
|
|
80
81
|
* :attr:`~eodag.config.PluginConfig.order_method` (``str``) HTTP request method, ``GET`` (default) or ``POST``
|
|
81
|
-
* :attr:`~eodag.config.PluginConfig.order_headers` (``[
|
|
82
|
+
* :attr:`~eodag.config.PluginConfig.order_headers` (``[dict[str, str]]``): order request headers
|
|
82
83
|
* :attr:`~eodag.config.PluginConfig.order_on_response` (:class:`~eodag.config.PluginConfig.OrderOnResponse`):
|
|
83
84
|
a typed dictionary containing the key :attr:`~eodag.config.PluginConfig.OrderOnResponse.metadata_mapping`
|
|
84
85
|
which can be used to add new product properties based on the data in response to the order request
|
|
@@ -93,7 +94,7 @@ class S3RestDownload(Download):
|
|
|
93
94
|
def download(
|
|
94
95
|
self,
|
|
95
96
|
product: EOProduct,
|
|
96
|
-
auth: Optional[Union[AuthBase,
|
|
97
|
+
auth: Optional[Union[AuthBase, S3SessionKwargs]] = None,
|
|
97
98
|
progress_callback: Optional[ProgressCallback] = None,
|
|
98
99
|
wait: float = DEFAULT_DOWNLOAD_WAIT,
|
|
99
100
|
timeout: float = DEFAULT_DOWNLOAD_TIMEOUT,
|
|
@@ -270,7 +271,7 @@ class S3RestDownload(Download):
|
|
|
270
271
|
os.remove(record_filename)
|
|
271
272
|
|
|
272
273
|
# total size for progress_callback
|
|
273
|
-
size_list:
|
|
274
|
+
size_list: list[int] = [
|
|
274
275
|
int(node.firstChild.nodeValue) # type: ignore[attr-defined]
|
|
275
276
|
for node in xmldoc.getElementsByTagName("Size")
|
|
276
277
|
if node.firstChild is not None
|
eodag/plugins/manager.py
CHANGED
|
@@ -21,18 +21,7 @@ import logging
|
|
|
21
21
|
import re
|
|
22
22
|
from operator import attrgetter
|
|
23
23
|
from pathlib import Path
|
|
24
|
-
from typing import
|
|
25
|
-
TYPE_CHECKING,
|
|
26
|
-
Any,
|
|
27
|
-
Dict,
|
|
28
|
-
Iterator,
|
|
29
|
-
List,
|
|
30
|
-
Optional,
|
|
31
|
-
Tuple,
|
|
32
|
-
Type,
|
|
33
|
-
Union,
|
|
34
|
-
cast,
|
|
35
|
-
)
|
|
24
|
+
from typing import TYPE_CHECKING, Any, Iterator, Optional, Union, cast
|
|
36
25
|
|
|
37
26
|
import pkg_resources
|
|
38
27
|
|
|
@@ -61,6 +50,7 @@ if TYPE_CHECKING:
|
|
|
61
50
|
from eodag.api.product import EOProduct
|
|
62
51
|
from eodag.config import PluginConfig, ProviderConfig
|
|
63
52
|
from eodag.plugins.base import PluginTopic
|
|
53
|
+
from eodag.types import S3SessionKwargs
|
|
64
54
|
|
|
65
55
|
|
|
66
56
|
logger = logging.getLogger("eodag.plugins.manager")
|
|
@@ -84,11 +74,11 @@ class PluginManager:
|
|
|
84
74
|
|
|
85
75
|
supported_topics = set(PLUGINS_TOPICS_KEYS)
|
|
86
76
|
|
|
87
|
-
product_type_to_provider_config_map:
|
|
77
|
+
product_type_to_provider_config_map: dict[str, list[ProviderConfig]]
|
|
88
78
|
|
|
89
|
-
skipped_plugins:
|
|
79
|
+
skipped_plugins: list[str]
|
|
90
80
|
|
|
91
|
-
def __init__(self, providers_config:
|
|
81
|
+
def __init__(self, providers_config: dict[str, ProviderConfig]) -> None:
|
|
92
82
|
self.skipped_plugins = []
|
|
93
83
|
self.providers_config = providers_config
|
|
94
84
|
# Load all the plugins. This will make all plugin classes of a particular
|
|
@@ -144,14 +134,14 @@ class PluginManager:
|
|
|
144
134
|
self.rebuild()
|
|
145
135
|
|
|
146
136
|
def rebuild(
|
|
147
|
-
self, providers_config: Optional[
|
|
137
|
+
self, providers_config: Optional[dict[str, ProviderConfig]] = None
|
|
148
138
|
) -> None:
|
|
149
139
|
"""(Re)Build plugin manager mapping and cache"""
|
|
150
140
|
if providers_config is not None:
|
|
151
141
|
self.providers_config = providers_config
|
|
152
142
|
|
|
153
143
|
self.build_product_type_to_provider_config_map()
|
|
154
|
-
self._built_plugins_cache:
|
|
144
|
+
self._built_plugins_cache: dict[tuple[str, str, str], Any] = {}
|
|
155
145
|
|
|
156
146
|
def build_product_type_to_provider_config_map(self) -> None:
|
|
157
147
|
"""Build mapping conf between product types and providers"""
|
|
@@ -211,7 +201,7 @@ class PluginManager:
|
|
|
211
201
|
)
|
|
212
202
|
return plugin
|
|
213
203
|
|
|
214
|
-
configs: Optional[
|
|
204
|
+
configs: Optional[list[ProviderConfig]]
|
|
215
205
|
if product_type:
|
|
216
206
|
configs = self.product_type_to_provider_config_map.get(product_type)
|
|
217
207
|
if not configs:
|
|
@@ -378,7 +368,7 @@ class PluginManager:
|
|
|
378
368
|
provider: str,
|
|
379
369
|
matching_url: Optional[str] = None,
|
|
380
370
|
matching_conf: Optional[PluginConfig] = None,
|
|
381
|
-
) -> Optional[Union[AuthBase,
|
|
371
|
+
) -> Optional[Union[AuthBase, S3SessionKwargs]]:
|
|
382
372
|
"""Authenticate and return the authenticated object for the first matching
|
|
383
373
|
authentication plugin
|
|
384
374
|
|
|
@@ -447,7 +437,7 @@ class PluginManager:
|
|
|
447
437
|
self,
|
|
448
438
|
provider: str,
|
|
449
439
|
plugin_conf: PluginConfig,
|
|
450
|
-
topic_class:
|
|
440
|
+
topic_class: type[PluginTopic],
|
|
451
441
|
) -> Union[Api, Search, Download, Authentication, Crunch]:
|
|
452
442
|
"""Build the plugin of the given topic with the given plugin configuration and
|
|
453
443
|
registered as the given provider
|
eodag/plugins/search/__init__.py
CHANGED
|
@@ -24,11 +24,12 @@ from typing import TYPE_CHECKING
|
|
|
24
24
|
from eodag.utils import DEFAULT_ITEMS_PER_PAGE, DEFAULT_PAGE
|
|
25
25
|
|
|
26
26
|
if TYPE_CHECKING:
|
|
27
|
-
from typing import Any,
|
|
27
|
+
from typing import Any, Optional, Union
|
|
28
28
|
|
|
29
29
|
from requests.auth import AuthBase
|
|
30
30
|
|
|
31
31
|
from eodag.plugins.authentication.base import Authentication
|
|
32
|
+
from eodag.types import S3SessionKwargs
|
|
32
33
|
|
|
33
34
|
|
|
34
35
|
@dataclass
|
|
@@ -38,7 +39,7 @@ class PreparedSearch:
|
|
|
38
39
|
product_type: Optional[str] = None
|
|
39
40
|
page: Optional[int] = DEFAULT_PAGE
|
|
40
41
|
items_per_page: Optional[int] = DEFAULT_ITEMS_PER_PAGE
|
|
41
|
-
auth: Optional[Union[AuthBase,
|
|
42
|
+
auth: Optional[Union[AuthBase, S3SessionKwargs]] = None
|
|
42
43
|
auth_plugin: Optional[Authentication] = None
|
|
43
44
|
count: bool = True
|
|
44
45
|
url: Optional[str] = None
|
|
@@ -46,9 +47,9 @@ class PreparedSearch:
|
|
|
46
47
|
exception_message: Optional[str] = None
|
|
47
48
|
|
|
48
49
|
need_count: bool = field(init=False, repr=False)
|
|
49
|
-
query_params:
|
|
50
|
+
query_params: dict[str, Any] = field(init=False, repr=False)
|
|
50
51
|
query_string: str = field(init=False, repr=False)
|
|
51
|
-
search_urls:
|
|
52
|
-
product_type_def_params:
|
|
52
|
+
search_urls: list[str] = field(init=False, repr=False)
|
|
53
|
+
product_type_def_params: dict[str, Any] = field(init=False, repr=False)
|
|
53
54
|
total_items_nb: int = field(init=False, repr=False)
|
|
54
55
|
sort_by_qs: str = field(init=False, repr=False)
|
eodag/plugins/search/base.py
CHANGED
|
@@ -43,12 +43,13 @@ from eodag.utils import (
|
|
|
43
43
|
from eodag.utils.exceptions import ValidationError
|
|
44
44
|
|
|
45
45
|
if TYPE_CHECKING:
|
|
46
|
-
from typing import Any,
|
|
46
|
+
from typing import Any, Optional, Union
|
|
47
47
|
|
|
48
48
|
from requests.auth import AuthBase
|
|
49
49
|
|
|
50
50
|
from eodag.api.product import EOProduct
|
|
51
51
|
from eodag.config import PluginConfig
|
|
52
|
+
from eodag.types import S3SessionKwargs
|
|
52
53
|
|
|
53
54
|
logger = logging.getLogger("eodag.search.base")
|
|
54
55
|
|
|
@@ -60,9 +61,9 @@ class Search(PluginTopic):
|
|
|
60
61
|
:param config: An EODAG plugin configuration
|
|
61
62
|
"""
|
|
62
63
|
|
|
63
|
-
auth: Union[AuthBase,
|
|
64
|
+
auth: Union[AuthBase, S3SessionKwargs]
|
|
64
65
|
next_page_url: Optional[str]
|
|
65
|
-
next_page_query_obj: Optional[
|
|
66
|
+
next_page_query_obj: Optional[dict[str, Any]]
|
|
66
67
|
total_items_nb: int
|
|
67
68
|
need_count: bool
|
|
68
69
|
_request: Any # needed by deprecated load_stac_items
|
|
@@ -71,7 +72,7 @@ class Search(PluginTopic):
|
|
|
71
72
|
super(Search, self).__init__(provider, config)
|
|
72
73
|
# Prepare the metadata mapping
|
|
73
74
|
# Do a shallow copy, the structure is flat enough for this to be sufficient
|
|
74
|
-
metas:
|
|
75
|
+
metas: dict[str, Any] = DEFAULT_METADATA_MAPPING.copy()
|
|
75
76
|
# Update the defaults with the mapping value. This will add any new key
|
|
76
77
|
# added by the provider mapping that is not in the default metadata
|
|
77
78
|
if self.config.metadata_mapping:
|
|
@@ -90,7 +91,7 @@ class Search(PluginTopic):
|
|
|
90
91
|
self,
|
|
91
92
|
prep: PreparedSearch = PreparedSearch(),
|
|
92
93
|
**kwargs: Any,
|
|
93
|
-
) ->
|
|
94
|
+
) -> tuple[list[EOProduct], Optional[int]]:
|
|
94
95
|
"""Implementation of how the products must be searched goes here.
|
|
95
96
|
|
|
96
97
|
This method must return a tuple with (1) a list of :class:`~eodag.api.product._product.EOProduct` instances
|
|
@@ -99,13 +100,13 @@ class Search(PluginTopic):
|
|
|
99
100
|
"""
|
|
100
101
|
raise NotImplementedError("A Search plugin must implement a method named query")
|
|
101
102
|
|
|
102
|
-
def discover_product_types(self, **kwargs: Any) -> Optional[
|
|
103
|
+
def discover_product_types(self, **kwargs: Any) -> Optional[dict[str, Any]]:
|
|
103
104
|
"""Fetch product types list from provider using `discover_product_types` conf"""
|
|
104
105
|
return None
|
|
105
106
|
|
|
106
107
|
def discover_queryables(
|
|
107
108
|
self, **kwargs: Any
|
|
108
|
-
) -> Optional[
|
|
109
|
+
) -> Optional[dict[str, Annotated[Any, FieldInfo]]]:
|
|
109
110
|
"""Fetch queryables list from provider using :attr:`~eodag.config.PluginConfig.discover_queryables` conf
|
|
110
111
|
|
|
111
112
|
:param kwargs: additional filters for queryables (``productType`` and other search
|
|
@@ -118,7 +119,7 @@ class Search(PluginTopic):
|
|
|
118
119
|
|
|
119
120
|
def _get_defaults_as_queryables(
|
|
120
121
|
self, product_type: str
|
|
121
|
-
) ->
|
|
122
|
+
) -> dict[str, Annotated[Any, FieldInfo]]:
|
|
122
123
|
"""
|
|
123
124
|
Return given product type default settings as queryables
|
|
124
125
|
|
|
@@ -128,7 +129,7 @@ class Search(PluginTopic):
|
|
|
128
129
|
defaults = deepcopy(self.config.products.get(product_type, {}))
|
|
129
130
|
defaults.pop("metadata_mapping", None)
|
|
130
131
|
|
|
131
|
-
queryables:
|
|
132
|
+
queryables: dict[str, Annotated[Any, FieldInfo]] = {}
|
|
132
133
|
for parameter, value in defaults.items():
|
|
133
134
|
queryables[parameter] = Annotated[type(value), Field(default=value)]
|
|
134
135
|
return queryables
|
|
@@ -149,8 +150,8 @@ class Search(PluginTopic):
|
|
|
149
150
|
)
|
|
150
151
|
|
|
151
152
|
def get_product_type_def_params(
|
|
152
|
-
self, product_type: str,
|
|
153
|
-
) ->
|
|
153
|
+
self, product_type: str, format_variables: Optional[dict[str, Any]] = None
|
|
154
|
+
) -> dict[str, Any]:
|
|
154
155
|
"""Get the provider product type definition parameters and specific settings
|
|
155
156
|
|
|
156
157
|
:param product_type: the desired product type
|
|
@@ -170,7 +171,8 @@ class Search(PluginTopic):
|
|
|
170
171
|
return {
|
|
171
172
|
k: v
|
|
172
173
|
for k, v in format_dict_items(
|
|
173
|
-
self.config.products[GENERIC_PRODUCT_TYPE],
|
|
174
|
+
self.config.products[GENERIC_PRODUCT_TYPE],
|
|
175
|
+
**(format_variables or {}),
|
|
174
176
|
).items()
|
|
175
177
|
if v
|
|
176
178
|
}
|
|
@@ -200,7 +202,7 @@ class Search(PluginTopic):
|
|
|
200
202
|
|
|
201
203
|
def get_metadata_mapping(
|
|
202
204
|
self, product_type: Optional[str] = None
|
|
203
|
-
) ->
|
|
205
|
+
) -> dict[str, Union[str, list[str]]]:
|
|
204
206
|
"""Get the plugin metadata mapping configuration (product type specific if exists)
|
|
205
207
|
|
|
206
208
|
:param product_type: the desired product type
|
|
@@ -212,7 +214,7 @@ class Search(PluginTopic):
|
|
|
212
214
|
)
|
|
213
215
|
return self.config.metadata_mapping
|
|
214
216
|
|
|
215
|
-
def get_sort_by_arg(self, kwargs:
|
|
217
|
+
def get_sort_by_arg(self, kwargs: dict[str, Any]) -> Optional[SortByList]:
|
|
216
218
|
"""Extract the ``sort_by`` argument from the kwargs or the provider default sort configuration
|
|
217
219
|
|
|
218
220
|
:param kwargs: Search arguments
|
|
@@ -233,7 +235,7 @@ class Search(PluginTopic):
|
|
|
233
235
|
|
|
234
236
|
def build_sort_by(
|
|
235
237
|
self, sort_by_arg: SortByList
|
|
236
|
-
) ->
|
|
238
|
+
) -> tuple[str, dict[str, list[dict[str, str]]]]:
|
|
237
239
|
"""Build the sorting part of the query string or body by transforming
|
|
238
240
|
the ``sort_by`` argument into a provider-specific string or dictionary
|
|
239
241
|
|
|
@@ -247,9 +249,9 @@ class Search(PluginTopic):
|
|
|
247
249
|
sort_by_arg = list(dict.fromkeys(sort_by_arg))
|
|
248
250
|
|
|
249
251
|
sort_by_qs: str = ""
|
|
250
|
-
sort_by_qp:
|
|
252
|
+
sort_by_qp: dict[str, Any] = {}
|
|
251
253
|
|
|
252
|
-
provider_sort_by_tuples_used:
|
|
254
|
+
provider_sort_by_tuples_used: list[tuple[str, str]] = []
|
|
253
255
|
for eodag_sort_by_tuple in sort_by_arg:
|
|
254
256
|
eodag_sort_param = eodag_sort_by_tuple[0]
|
|
255
257
|
provider_sort_param = self.config.sort["sort_param_mapping"].get(
|
|
@@ -282,7 +284,7 @@ class Search(PluginTopic):
|
|
|
282
284
|
if eodag_sort_order == "ASC"
|
|
283
285
|
else self.config.sort["sort_order_mapping"]["descending"]
|
|
284
286
|
)
|
|
285
|
-
provider_sort_by_tuple:
|
|
287
|
+
provider_sort_by_tuple: tuple[str, str] = (
|
|
286
288
|
provider_sort_param,
|
|
287
289
|
provider_sort_order,
|
|
288
290
|
)
|
|
@@ -315,7 +317,7 @@ class Search(PluginTopic):
|
|
|
315
317
|
sort_order=provider_sort_by_tuple[1],
|
|
316
318
|
)
|
|
317
319
|
try:
|
|
318
|
-
parsed_sort_by_tpl_dict:
|
|
320
|
+
parsed_sort_by_tpl_dict: dict[str, Any] = orjson.loads(
|
|
319
321
|
parsed_sort_by_tpl
|
|
320
322
|
)
|
|
321
323
|
sort_by_qp = update_nested_dict(
|
|
@@ -326,23 +328,25 @@ class Search(PluginTopic):
|
|
|
326
328
|
return (sort_by_qs, sort_by_qp)
|
|
327
329
|
|
|
328
330
|
def _get_product_type_queryables(
|
|
329
|
-
self, product_type: Optional[str], alias: Optional[str], filters:
|
|
330
|
-
) ->
|
|
331
|
-
default_values:
|
|
331
|
+
self, product_type: Optional[str], alias: Optional[str], filters: dict[str, Any]
|
|
332
|
+
) -> QueryablesDict:
|
|
333
|
+
default_values: dict[str, Any] = deepcopy(
|
|
332
334
|
getattr(self.config, "products", {}).get(product_type, {})
|
|
333
335
|
)
|
|
334
336
|
default_values.pop("metadata_mapping", None)
|
|
335
337
|
try:
|
|
336
338
|
filters["productType"] = product_type
|
|
337
|
-
|
|
339
|
+
queryables = self.discover_queryables(**{**default_values, **filters}) or {}
|
|
338
340
|
except NotImplementedError:
|
|
339
|
-
|
|
341
|
+
queryables = self.queryables_from_metadata_mapping(product_type, alias)
|
|
342
|
+
|
|
343
|
+
return QueryablesDict(**queryables)
|
|
340
344
|
|
|
341
345
|
def list_queryables(
|
|
342
346
|
self,
|
|
343
|
-
filters:
|
|
344
|
-
available_product_types:
|
|
345
|
-
product_type_configs:
|
|
347
|
+
filters: dict[str, Any],
|
|
348
|
+
available_product_types: list[Any],
|
|
349
|
+
product_type_configs: dict[str, dict[str, Any]],
|
|
346
350
|
product_type: Optional[str] = None,
|
|
347
351
|
alias: Optional[str] = None,
|
|
348
352
|
) -> QueryablesDict:
|
|
@@ -369,19 +373,11 @@ class Search(PluginTopic):
|
|
|
369
373
|
if product_type:
|
|
370
374
|
self.config.product_type_config = product_type_configs[product_type]
|
|
371
375
|
queryables = self._get_product_type_queryables(product_type, alias, filters)
|
|
372
|
-
|
|
373
|
-
|
|
374
|
-
|
|
375
|
-
additional_properties = False
|
|
376
|
-
else:
|
|
377
|
-
additional_properties = True
|
|
378
|
-
return QueryablesDict(
|
|
379
|
-
additional_properties=additional_properties,
|
|
380
|
-
additional_information=additional_info,
|
|
381
|
-
**queryables,
|
|
382
|
-
)
|
|
376
|
+
queryables.additional_information = additional_info
|
|
377
|
+
|
|
378
|
+
return queryables
|
|
383
379
|
else:
|
|
384
|
-
all_queryables:
|
|
380
|
+
all_queryables: dict[str, Any] = {}
|
|
385
381
|
for pt in available_product_types:
|
|
386
382
|
self.config.product_type_config = product_type_configs[pt]
|
|
387
383
|
pt_queryables = self._get_product_type_queryables(pt, None, filters)
|
|
@@ -399,18 +395,18 @@ class Search(PluginTopic):
|
|
|
399
395
|
|
|
400
396
|
def queryables_from_metadata_mapping(
|
|
401
397
|
self, product_type: Optional[str] = None, alias: Optional[str] = None
|
|
402
|
-
) ->
|
|
398
|
+
) -> dict[str, Annotated[Any, FieldInfo]]:
|
|
403
399
|
"""
|
|
404
400
|
Extract queryable parameters from product type metadata mapping.
|
|
405
401
|
:param product_type: product type id (optional)
|
|
406
402
|
:param alias: (optional) alias of the product type
|
|
407
403
|
:returns: dict of annotated queryables
|
|
408
404
|
"""
|
|
409
|
-
metadata_mapping:
|
|
405
|
+
metadata_mapping: dict[str, Any] = deepcopy(
|
|
410
406
|
self.get_metadata_mapping(product_type)
|
|
411
407
|
)
|
|
412
408
|
|
|
413
|
-
queryables:
|
|
409
|
+
queryables: dict[str, Annotated[Any, FieldInfo]] = {}
|
|
414
410
|
|
|
415
411
|
for param in list(metadata_mapping.keys()):
|
|
416
412
|
if NOT_MAPPED in metadata_mapping[param] or not isinstance(
|