eodag 3.1.0b1__py3-none-any.whl → 3.1.0b2__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- eodag/api/core.py +59 -52
- eodag/api/product/_assets.py +5 -5
- eodag/api/product/_product.py +27 -12
- eodag/api/product/drivers/__init__.py +81 -4
- eodag/api/product/drivers/base.py +65 -4
- eodag/api/product/drivers/generic.py +65 -0
- eodag/api/product/drivers/sentinel1.py +97 -0
- eodag/api/product/drivers/sentinel2.py +95 -0
- eodag/api/product/metadata_mapping.py +62 -74
- eodag/api/search_result.py +13 -23
- eodag/cli.py +4 -4
- eodag/config.py +66 -69
- eodag/plugins/apis/base.py +1 -1
- eodag/plugins/apis/ecmwf.py +10 -9
- eodag/plugins/apis/usgs.py +11 -10
- eodag/plugins/authentication/aws_auth.py +16 -13
- eodag/plugins/authentication/base.py +5 -3
- eodag/plugins/authentication/header.py +3 -3
- eodag/plugins/authentication/keycloak.py +4 -4
- eodag/plugins/authentication/oauth.py +7 -3
- eodag/plugins/authentication/openid_connect.py +14 -14
- eodag/plugins/authentication/sas_auth.py +4 -4
- eodag/plugins/authentication/token.py +7 -7
- eodag/plugins/authentication/token_exchange.py +1 -1
- eodag/plugins/base.py +4 -4
- eodag/plugins/crunch/base.py +4 -4
- eodag/plugins/crunch/filter_date.py +4 -4
- eodag/plugins/crunch/filter_latest_intersect.py +6 -6
- eodag/plugins/crunch/filter_latest_tpl_name.py +7 -7
- eodag/plugins/crunch/filter_overlap.py +4 -4
- eodag/plugins/crunch/filter_property.py +4 -4
- eodag/plugins/download/aws.py +47 -66
- eodag/plugins/download/base.py +8 -17
- eodag/plugins/download/creodias_s3.py +2 -2
- eodag/plugins/download/http.py +30 -32
- eodag/plugins/download/s3rest.py +5 -4
- eodag/plugins/manager.py +10 -20
- eodag/plugins/search/__init__.py +6 -5
- eodag/plugins/search/base.py +35 -40
- eodag/plugins/search/build_search_result.py +69 -68
- eodag/plugins/search/cop_marine.py +22 -12
- eodag/plugins/search/creodias_s3.py +8 -78
- eodag/plugins/search/csw.py +11 -11
- eodag/plugins/search/data_request_search.py +16 -15
- eodag/plugins/search/qssearch.py +56 -52
- eodag/plugins/search/stac_list_assets.py +85 -0
- eodag/plugins/search/static_stac_search.py +3 -3
- eodag/resources/ext_product_types.json +1 -1
- eodag/resources/product_types.yml +288 -288
- eodag/resources/providers.yml +146 -6
- eodag/resources/stac_api.yml +2 -2
- eodag/resources/user_conf_template.yml +11 -0
- eodag/rest/cache.py +2 -2
- eodag/rest/config.py +3 -3
- eodag/rest/core.py +24 -24
- eodag/rest/errors.py +5 -5
- eodag/rest/server.py +3 -11
- eodag/rest/stac.py +40 -38
- eodag/rest/types/collections_search.py +3 -3
- eodag/rest/types/eodag_search.py +23 -23
- eodag/rest/types/queryables.py +13 -13
- eodag/rest/types/stac_search.py +15 -25
- eodag/rest/utils/__init__.py +11 -21
- eodag/rest/utils/cql_evaluate.py +6 -6
- eodag/rest/utils/rfc3339.py +2 -2
- eodag/types/__init__.py +24 -18
- eodag/types/bbox.py +2 -2
- eodag/types/download_args.py +2 -2
- eodag/types/queryables.py +5 -2
- eodag/types/search_args.py +4 -4
- eodag/types/whoosh.py +1 -3
- eodag/utils/__init__.py +81 -40
- eodag/utils/exceptions.py +2 -2
- eodag/utils/import_system.py +2 -2
- eodag/utils/requests.py +2 -2
- eodag/utils/rest.py +2 -2
- eodag/utils/s3.py +208 -0
- eodag/utils/stac_reader.py +10 -10
- {eodag-3.1.0b1.dist-info → eodag-3.1.0b2.dist-info}/METADATA +5 -4
- eodag-3.1.0b2.dist-info/RECORD +113 -0
- {eodag-3.1.0b1.dist-info → eodag-3.1.0b2.dist-info}/entry_points.txt +1 -0
- eodag-3.1.0b1.dist-info/RECORD +0 -108
- {eodag-3.1.0b1.dist-info → eodag-3.1.0b2.dist-info}/LICENSE +0 -0
- {eodag-3.1.0b1.dist-info → eodag-3.1.0b2.dist-info}/WHEEL +0 -0
- {eodag-3.1.0b1.dist-info → eodag-3.1.0b2.dist-info}/top_level.txt +0 -0
|
@@ -20,7 +20,7 @@ from __future__ import annotations
|
|
|
20
20
|
import logging
|
|
21
21
|
import time
|
|
22
22
|
from datetime import datetime, timedelta, timezone
|
|
23
|
-
from typing import TYPE_CHECKING, Any,
|
|
23
|
+
from typing import TYPE_CHECKING, Any, Optional, cast
|
|
24
24
|
|
|
25
25
|
import requests
|
|
26
26
|
|
|
@@ -36,6 +36,7 @@ from eodag.utils import (
|
|
|
36
36
|
DEFAULT_ITEMS_PER_PAGE,
|
|
37
37
|
DEFAULT_MISSION_START_DATE,
|
|
38
38
|
DEFAULT_PAGE,
|
|
39
|
+
DEFAULT_SEARCH_TIMEOUT,
|
|
39
40
|
GENERIC_PRODUCT_TYPE,
|
|
40
41
|
HTTP_REQ_TIMEOUT,
|
|
41
42
|
USER_AGENT,
|
|
@@ -113,10 +114,10 @@ class DataRequestSearch(Search):
|
|
|
113
114
|
* :attr:`~eodag.config.PluginConfig.DiscoverProductTypes.generic_product_type_id` (``str``): mapping for the
|
|
114
115
|
product type id
|
|
115
116
|
* :attr:`~eodag.config.PluginConfig.DiscoverProductTypes.generic_product_type_parsable_metadata`
|
|
116
|
-
(``
|
|
117
|
+
(``dict[str, str]``): mapping for product type metadata (e.g. ``abstract``, ``licence``) which can be parsed
|
|
117
118
|
from the provider result
|
|
118
119
|
* :attr:`~eodag.config.PluginConfig.DiscoverProductTypes.generic_product_type_parsable_properties`
|
|
119
|
-
(``
|
|
120
|
+
(``dict[str, str]``): mapping for product type properties which can be parsed from the result and are not
|
|
120
121
|
product type metadata
|
|
121
122
|
* :attr:`~eodag.config.PluginConfig.DiscoverProductTypes.single_collection_fetch_url` (``str``): url to fetch
|
|
122
123
|
data for a single collection; used if product type metadata is not available from the endpoint given in
|
|
@@ -125,7 +126,7 @@ class DataRequestSearch(Search):
|
|
|
125
126
|
to be added to the :attr:`~eodag.config.PluginConfig.DiscoverProductTypes.fetch_url` to filter for a
|
|
126
127
|
collection
|
|
127
128
|
* :attr:`~eodag.config.PluginConfig.DiscoverProductTypes.single_product_type_parsable_metadata`
|
|
128
|
-
(``
|
|
129
|
+
(``dict[str, str]``): mapping for product type metadata returned by the endpoint given in
|
|
129
130
|
:attr:`~eodag.config.PluginConfig.DiscoverProductTypes.single_collection_fetch_url`.
|
|
130
131
|
|
|
131
132
|
* :attr:`~eodag.config.PluginConfig.constraints_file_url` (``str``): url to fetch the constraints for a specific
|
|
@@ -133,7 +134,7 @@ class DataRequestSearch(Search):
|
|
|
133
134
|
* :attr:`~eodag.config.PluginConfig.constraints_entry` (``str``): key in the json result where the constraints
|
|
134
135
|
can be found; if not given, it is assumed that the constraints are on top level of the result, i.e.
|
|
135
136
|
the result is an array of constraints
|
|
136
|
-
* :attr:`~eodag.config.PluginConfig.metadata_mapping` (``
|
|
137
|
+
* :attr:`~eodag.config.PluginConfig.metadata_mapping` (``dict[str, Any]``): The search plugins of this kind can
|
|
137
138
|
detect when a metadata mapping is "query-able", and get the semantics of how to format the query string
|
|
138
139
|
parameter that enables to make a query on the corresponding metadata. To make a metadata query-able,
|
|
139
140
|
just configure it in the metadata mapping to be a list of 2 items, the first one being the
|
|
@@ -207,10 +208,10 @@ class DataRequestSearch(Search):
|
|
|
207
208
|
self.config.pagination["next_page_url_key_path"] = string_to_jsonpath(
|
|
208
209
|
self.config.pagination.get("next_page_url_key_path", None)
|
|
209
210
|
)
|
|
210
|
-
self.download_info:
|
|
211
|
+
self.download_info: dict[str, Any] = {}
|
|
211
212
|
self.data_request_id = None
|
|
212
213
|
|
|
213
|
-
def discover_product_types(self, **kwargs: Any) -> Optional[
|
|
214
|
+
def discover_product_types(self, **kwargs: Any) -> Optional[dict[str, Any]]:
|
|
214
215
|
"""Fetch product types is disabled for `DataRequestSearch`
|
|
215
216
|
|
|
216
217
|
:returns: empty dict
|
|
@@ -226,7 +227,7 @@ class DataRequestSearch(Search):
|
|
|
226
227
|
self,
|
|
227
228
|
prep: PreparedSearch = PreparedSearch(),
|
|
228
229
|
**kwargs: Any,
|
|
229
|
-
) ->
|
|
230
|
+
) -> tuple[list[EOProduct], Optional[int]]:
|
|
230
231
|
"""
|
|
231
232
|
performs the search for a provider where several steps are required to fetch the data
|
|
232
233
|
"""
|
|
@@ -308,7 +309,7 @@ class DataRequestSearch(Search):
|
|
|
308
309
|
request_finished = True
|
|
309
310
|
|
|
310
311
|
# loop to check search job status
|
|
311
|
-
search_timeout = int(getattr(self.config, "timeout",
|
|
312
|
+
search_timeout = int(getattr(self.config, "timeout", DEFAULT_SEARCH_TIMEOUT))
|
|
312
313
|
logger.info(
|
|
313
314
|
f"checking status of request job {data_request_id} (timeout={search_timeout}s)"
|
|
314
315
|
)
|
|
@@ -431,7 +432,7 @@ class DataRequestSearch(Search):
|
|
|
431
432
|
|
|
432
433
|
def _get_result_data(
|
|
433
434
|
self, data_request_id: str, items_per_page: int, page: int
|
|
434
|
-
) ->
|
|
435
|
+
) -> dict[str, Any]:
|
|
435
436
|
page = page - 1 + self.config.pagination.get("start_page", 1)
|
|
436
437
|
url = self.config.result_url.format(
|
|
437
438
|
jobId=data_request_id, items_per_page=items_per_page, page=page
|
|
@@ -450,18 +451,18 @@ class DataRequestSearch(Search):
|
|
|
450
451
|
|
|
451
452
|
def _convert_result_data(
|
|
452
453
|
self,
|
|
453
|
-
result_data:
|
|
454
|
+
result_data: dict[str, Any],
|
|
454
455
|
data_request_id: str,
|
|
455
456
|
product_type: str,
|
|
456
457
|
**kwargs: Any,
|
|
457
|
-
) ->
|
|
458
|
+
) -> tuple[list[EOProduct], int]:
|
|
458
459
|
"""Build EOProducts from provider results"""
|
|
459
460
|
results_entry = self.config.results_entry
|
|
460
461
|
results = result_data[results_entry]
|
|
461
462
|
logger.debug(
|
|
462
463
|
"Adapting %s plugin results to eodag product representation" % len(results)
|
|
463
464
|
)
|
|
464
|
-
products:
|
|
465
|
+
products: list[EOProduct] = []
|
|
465
466
|
for result in results:
|
|
466
467
|
product = EOProduct(
|
|
467
468
|
self.provider,
|
|
@@ -517,8 +518,8 @@ class DataRequestSearch(Search):
|
|
|
517
518
|
return False
|
|
518
519
|
|
|
519
520
|
def _apply_additional_filters(
|
|
520
|
-
self, result:
|
|
521
|
-
) ->
|
|
521
|
+
self, result: dict[str, Any], custom_filters: dict[str, str]
|
|
522
|
+
) -> dict[str, Any]:
|
|
522
523
|
filtered_result = []
|
|
523
524
|
results_entry = self.config.results_entry
|
|
524
525
|
results = result[results_entry]
|
eodag/plugins/search/qssearch.py
CHANGED
|
@@ -26,12 +26,8 @@ from typing import (
|
|
|
26
26
|
Annotated,
|
|
27
27
|
Any,
|
|
28
28
|
Callable,
|
|
29
|
-
Dict,
|
|
30
|
-
List,
|
|
31
29
|
Optional,
|
|
32
30
|
Sequence,
|
|
33
|
-
Set,
|
|
34
|
-
Tuple,
|
|
35
31
|
TypedDict,
|
|
36
32
|
cast,
|
|
37
33
|
get_args,
|
|
@@ -78,6 +74,7 @@ from eodag.types import json_field_definition_to_python, model_fields_to_annotat
|
|
|
78
74
|
from eodag.types.search_args import SortByList
|
|
79
75
|
from eodag.utils import (
|
|
80
76
|
DEFAULT_MISSION_START_DATE,
|
|
77
|
+
DEFAULT_SEARCH_TIMEOUT,
|
|
81
78
|
GENERIC_PRODUCT_TYPE,
|
|
82
79
|
HTTP_REQ_TIMEOUT,
|
|
83
80
|
REQ_RETRY_BACKOFF_FACTOR,
|
|
@@ -128,7 +125,7 @@ class QueryStringSearch(Search):
|
|
|
128
125
|
authentication error; only used if ``need_auth=true``
|
|
129
126
|
* :attr:`~eodag.config.PluginConfig.ssl_verify` (``bool``): if the ssl certificates should be verified in
|
|
130
127
|
requests; default: ``True``
|
|
131
|
-
* :attr:`~eodag.config.PluginConfig.dont_quote` (``
|
|
128
|
+
* :attr:`~eodag.config.PluginConfig.dont_quote` (``list[str]``): characters that should not be quoted in the
|
|
132
129
|
url params
|
|
133
130
|
* :attr:`~eodag.config.PluginConfig.timeout` (``int``): time to wait until request timeout in seconds;
|
|
134
131
|
default: ``5``
|
|
@@ -136,10 +133,10 @@ class QueryStringSearch(Search):
|
|
|
136
133
|
total number of retries to allow; default: ``3``
|
|
137
134
|
* :attr:`~eodag.config.PluginConfig.retry_backoff_factor` (``int``): :class:`urllib3.util.Retry`
|
|
138
135
|
``backoff_factor`` parameter, backoff factor to apply between attempts after the second try; default: ``2``
|
|
139
|
-
* :attr:`~eodag.config.PluginConfig.retry_status_forcelist` (``
|
|
136
|
+
* :attr:`~eodag.config.PluginConfig.retry_status_forcelist` (``list[int]``): :class:`urllib3.util.Retry`
|
|
140
137
|
``status_forcelist`` parameter, list of integer HTTP status codes that we should force a retry on; default:
|
|
141
138
|
``[401, 429, 500, 502, 503, 504]``
|
|
142
|
-
* :attr:`~eodag.config.PluginConfig.literal_search_params` (``
|
|
139
|
+
* :attr:`~eodag.config.PluginConfig.literal_search_params` (``dict[str, str]``): A mapping of (search_param =>
|
|
143
140
|
search_value) pairs giving search parameters to be passed as is in the search url query string. This is useful
|
|
144
141
|
for example in situations where the user wants to add a fixed search query parameter exactly
|
|
145
142
|
as it is done on the provider interface.
|
|
@@ -183,13 +180,13 @@ class QueryStringSearch(Search):
|
|
|
183
180
|
* :attr:`~eodag.config.PluginConfig.DiscoverProductTypes.generic_product_type_id` (``str``): mapping for the
|
|
184
181
|
product type id
|
|
185
182
|
* :attr:`~eodag.config.PluginConfig.DiscoverProductTypes.generic_product_type_parsable_metadata`
|
|
186
|
-
(``
|
|
183
|
+
(``dict[str, str]``): mapping for product type metadata (e.g. ``abstract``, ``licence``) which can be parsed
|
|
187
184
|
from the provider result
|
|
188
185
|
* :attr:`~eodag.config.PluginConfig.DiscoverProductTypes.generic_product_type_parsable_properties`
|
|
189
|
-
(``
|
|
186
|
+
(``dict[str, str]``): mapping for product type properties which can be parsed from the result and are not
|
|
190
187
|
product type metadata
|
|
191
188
|
* :attr:`~eodag.config.PluginConfig.DiscoverProductTypes.generic_product_type_unparsable_properties`
|
|
192
|
-
(``
|
|
189
|
+
(``dict[str, str]``): mapping for product type properties which cannot be parsed from the result and are not
|
|
193
190
|
product type metadata
|
|
194
191
|
* :attr:`~eodag.config.PluginConfig.DiscoverProductTypes.single_collection_fetch_url` (``str``): url to fetch
|
|
195
192
|
data for a single collection; used if product type metadata is not available from the endpoint given in
|
|
@@ -198,13 +195,13 @@ class QueryStringSearch(Search):
|
|
|
198
195
|
to be added to the :attr:`~eodag.config.PluginConfig.DiscoverProductTypes.fetch_url` to filter for a
|
|
199
196
|
collection
|
|
200
197
|
* :attr:`~eodag.config.PluginConfig.DiscoverProductTypes.single_product_type_parsable_metadata`
|
|
201
|
-
(``
|
|
198
|
+
(``dict[str, str]``): mapping for product type metadata returned by the endpoint given in
|
|
202
199
|
:attr:`~eodag.config.PluginConfig.DiscoverProductTypes.single_collection_fetch_url`.
|
|
203
200
|
|
|
204
201
|
* :attr:`~eodag.config.PluginConfig.sort` (:class:`~eodag.config.PluginConfig.Sort`): configuration for sorting
|
|
205
202
|
the results. It contains the keys:
|
|
206
203
|
|
|
207
|
-
* :attr:`~eodag.config.PluginConfig.Sort.sort_by_default` (``
|
|
204
|
+
* :attr:`~eodag.config.PluginConfig.Sort.sort_by_default` (``list[Tuple(str, Literal["ASC", "DESC"])]``):
|
|
208
205
|
parameter and sort order by which the result will be sorted by default (if the user does not enter a
|
|
209
206
|
``sort_by`` parameter); if not given the result will use the default sorting of the provider; Attention:
|
|
210
207
|
for some providers sorting might cause a timeout if no filters are used. In that case no default
|
|
@@ -220,12 +217,12 @@ class QueryStringSearch(Search):
|
|
|
220
217
|
* :attr:`~eodag.config.PluginConfig.Sort.sort_param_mapping` (``Dict [str, str]``): mapping for the parameters
|
|
221
218
|
available for sorting
|
|
222
219
|
* :attr:`~eodag.config.PluginConfig.Sort.sort_order_mapping`
|
|
223
|
-
(``
|
|
220
|
+
(``dict[Literal["ascending", "descending"], str]``): mapping for the sort order
|
|
224
221
|
* :attr:`~eodag.config.PluginConfig.Sort.max_sort_params` (``int``): maximum number of sort parameters
|
|
225
222
|
supported by the provider; used to validate the user input to avoid failed requests or unexpected behaviour
|
|
226
223
|
(not all parameters are used in the request)
|
|
227
224
|
|
|
228
|
-
* :attr:`~eodag.config.PluginConfig.metadata_mapping` (``
|
|
225
|
+
* :attr:`~eodag.config.PluginConfig.metadata_mapping` (``dict[str, Any]``): The search plugins of this kind can
|
|
229
226
|
detect when a metadata mapping is "query-able", and get the semantics of how to format the query string
|
|
230
227
|
parameter that enables to make a query on the corresponding metadata. To make a metadata query-able,
|
|
231
228
|
just configure it in the metadata mapping to be a list of 2 items, the first one being the
|
|
@@ -258,7 +255,7 @@ class QueryStringSearch(Search):
|
|
|
258
255
|
metadata is activated; default: ``False``; if false, the other parameters are not used;
|
|
259
256
|
* :attr:`~eodag.config.PluginConfig.DiscoverMetadata.metadata_pattern` (``str``): regex string a parameter in
|
|
260
257
|
the result should match so that is used
|
|
261
|
-
* :attr:`~eodag.config.PluginConfig.DiscoverMetadata.search_param` (``Union [str,
|
|
258
|
+
* :attr:`~eodag.config.PluginConfig.DiscoverMetadata.search_param` (``Union [str, dict[str, Any]]``): format
|
|
262
259
|
to add a query param given by the user and not in the metadata mapping to the requests, 'metadata' will be
|
|
263
260
|
replaced by the search param; can be a string or a dict containing
|
|
264
261
|
:attr:`~eodag.config.PluginConfig.free_text_search_operations`
|
|
@@ -286,7 +283,7 @@ class QueryStringSearch(Search):
|
|
|
286
283
|
the result is an array of constraints
|
|
287
284
|
"""
|
|
288
285
|
|
|
289
|
-
extract_properties:
|
|
286
|
+
extract_properties: dict[str, Callable[..., dict[str, Any]]] = {
|
|
290
287
|
"xml": properties_from_xml,
|
|
291
288
|
"json": properties_from_json,
|
|
292
289
|
}
|
|
@@ -297,8 +294,8 @@ class QueryStringSearch(Search):
|
|
|
297
294
|
self.config.__dict__.setdefault("results_entry", "features")
|
|
298
295
|
self.config.__dict__.setdefault("pagination", {})
|
|
299
296
|
self.config.__dict__.setdefault("free_text_search_operations", {})
|
|
300
|
-
self.search_urls:
|
|
301
|
-
self.query_params:
|
|
297
|
+
self.search_urls: list[str] = []
|
|
298
|
+
self.query_params: dict[str, str] = dict()
|
|
302
299
|
self.query_string = ""
|
|
303
300
|
self.next_page_url = None
|
|
304
301
|
self.next_page_query_obj = None
|
|
@@ -443,7 +440,7 @@ class QueryStringSearch(Search):
|
|
|
443
440
|
self.next_page_query_obj = None
|
|
444
441
|
self.next_page_merge = None
|
|
445
442
|
|
|
446
|
-
def discover_product_types(self, **kwargs: Any) -> Optional[
|
|
443
|
+
def discover_product_types(self, **kwargs: Any) -> Optional[dict[str, Any]]:
|
|
447
444
|
"""Fetch product types list from provider using `discover_product_types` conf
|
|
448
445
|
|
|
449
446
|
:returns: configuration dict containing fetched product types information
|
|
@@ -460,7 +457,7 @@ class QueryStringSearch(Search):
|
|
|
460
457
|
# no pagination
|
|
461
458
|
return self.discover_product_types_per_page(**kwargs)
|
|
462
459
|
|
|
463
|
-
conf_update_dict:
|
|
460
|
+
conf_update_dict: dict[str, Any] = {
|
|
464
461
|
"providers_config": {},
|
|
465
462
|
"product_types_config": {},
|
|
466
463
|
}
|
|
@@ -493,7 +490,7 @@ class QueryStringSearch(Search):
|
|
|
493
490
|
|
|
494
491
|
def discover_product_types_per_page(
|
|
495
492
|
self, **kwargs: Any
|
|
496
|
-
) -> Optional[
|
|
493
|
+
) -> Optional[dict[str, Any]]:
|
|
497
494
|
"""Fetch product types list from provider using `discover_product_types` conf
|
|
498
495
|
using paginated ``kwargs["fetch_url"]``
|
|
499
496
|
|
|
@@ -551,7 +548,7 @@ class QueryStringSearch(Search):
|
|
|
551
548
|
return None
|
|
552
549
|
else:
|
|
553
550
|
try:
|
|
554
|
-
conf_update_dict:
|
|
551
|
+
conf_update_dict: dict[str, Any] = {
|
|
555
552
|
"providers_config": {},
|
|
556
553
|
"product_types_config": {},
|
|
557
554
|
}
|
|
@@ -570,7 +567,7 @@ class QueryStringSearch(Search):
|
|
|
570
567
|
result = result[0]
|
|
571
568
|
|
|
572
569
|
def conf_update_from_product_type_result(
|
|
573
|
-
product_type_result:
|
|
570
|
+
product_type_result: dict[str, Any]
|
|
574
571
|
) -> None:
|
|
575
572
|
"""Update ``conf_update_dict`` using given product type json response"""
|
|
576
573
|
# providers_config extraction
|
|
@@ -698,7 +695,7 @@ class QueryStringSearch(Search):
|
|
|
698
695
|
|
|
699
696
|
def _get_product_type_metadata_from_single_collection_endpoint(
|
|
700
697
|
self, product_type: str
|
|
701
|
-
) ->
|
|
698
|
+
) -> dict[str, Any]:
|
|
702
699
|
"""
|
|
703
700
|
retrieves additional product type information from an endpoint returning data for a single collection
|
|
704
701
|
:param product_type: product type
|
|
@@ -726,7 +723,7 @@ class QueryStringSearch(Search):
|
|
|
726
723
|
self,
|
|
727
724
|
prep: PreparedSearch = PreparedSearch(),
|
|
728
725
|
**kwargs: Any,
|
|
729
|
-
) ->
|
|
726
|
+
) -> tuple[list[EOProduct], Optional[int]]:
|
|
730
727
|
"""Perform a search on an OpenSearch-like interface
|
|
731
728
|
|
|
732
729
|
:param prep: Object collecting needed information for search.
|
|
@@ -806,14 +803,14 @@ class QueryStringSearch(Search):
|
|
|
806
803
|
reason="Simply run `self.config.metadata_mapping.update(metadata_mapping)` instead",
|
|
807
804
|
version="2.10.0",
|
|
808
805
|
)
|
|
809
|
-
def update_metadata_mapping(self, metadata_mapping:
|
|
806
|
+
def update_metadata_mapping(self, metadata_mapping: dict[str, Any]) -> None:
|
|
810
807
|
"""Update plugin metadata_mapping with input metadata_mapping configuration"""
|
|
811
808
|
if self.config.metadata_mapping:
|
|
812
809
|
self.config.metadata_mapping.update(metadata_mapping)
|
|
813
810
|
|
|
814
811
|
def build_query_string(
|
|
815
812
|
self, product_type: str, **kwargs: Any
|
|
816
|
-
) ->
|
|
813
|
+
) -> tuple[dict[str, Any], str]:
|
|
817
814
|
"""Build The query string using the search parameters"""
|
|
818
815
|
logger.debug("Building the query string that will be used for search")
|
|
819
816
|
query_params = format_query_params(product_type, self.config, kwargs)
|
|
@@ -832,7 +829,7 @@ class QueryStringSearch(Search):
|
|
|
832
829
|
self,
|
|
833
830
|
prep: PreparedSearch = PreparedSearch(page=None, items_per_page=None),
|
|
834
831
|
**kwargs: Any,
|
|
835
|
-
) ->
|
|
832
|
+
) -> tuple[list[str], Optional[int]]:
|
|
836
833
|
"""Build paginated urls"""
|
|
837
834
|
page = prep.page
|
|
838
835
|
items_per_page = prep.items_per_page
|
|
@@ -901,7 +898,7 @@ class QueryStringSearch(Search):
|
|
|
901
898
|
|
|
902
899
|
def do_search(
|
|
903
900
|
self, prep: PreparedSearch = PreparedSearch(items_per_page=None), **kwargs: Any
|
|
904
|
-
) ->
|
|
901
|
+
) -> list[Any]:
|
|
905
902
|
"""Perform the actual search request.
|
|
906
903
|
|
|
907
904
|
If there is a specified number of items per page, return the results as soon
|
|
@@ -918,7 +915,7 @@ class QueryStringSearch(Search):
|
|
|
918
915
|
"total_items_nb_key_path"
|
|
919
916
|
]
|
|
920
917
|
|
|
921
|
-
results:
|
|
918
|
+
results: list[Any] = []
|
|
922
919
|
for search_url in prep.search_urls:
|
|
923
920
|
single_search_prep = copy_copy(prep)
|
|
924
921
|
single_search_prep.url = search_url
|
|
@@ -1069,14 +1066,14 @@ class QueryStringSearch(Search):
|
|
|
1069
1066
|
|
|
1070
1067
|
def normalize_results(
|
|
1071
1068
|
self, results: RawSearchResult, **kwargs: Any
|
|
1072
|
-
) ->
|
|
1069
|
+
) -> list[EOProduct]:
|
|
1073
1070
|
"""Build EOProducts from provider results"""
|
|
1074
1071
|
normalize_remaining_count = len(results)
|
|
1075
1072
|
logger.debug(
|
|
1076
1073
|
"Adapting %s plugin results to eodag product representation"
|
|
1077
1074
|
% normalize_remaining_count
|
|
1078
1075
|
)
|
|
1079
|
-
products:
|
|
1076
|
+
products: list[EOProduct] = []
|
|
1080
1077
|
for result in results:
|
|
1081
1078
|
product = EOProduct(
|
|
1082
1079
|
self.provider,
|
|
@@ -1091,8 +1088,15 @@ class QueryStringSearch(Search):
|
|
|
1091
1088
|
product.properties = dict(
|
|
1092
1089
|
getattr(self.config, "product_type_config", {}), **product.properties
|
|
1093
1090
|
)
|
|
1094
|
-
# move assets from properties to product's attr
|
|
1095
|
-
product.
|
|
1091
|
+
# move assets from properties to product's attr, normalize keys & roles
|
|
1092
|
+
for key, asset in product.properties.pop("assets", {}).items():
|
|
1093
|
+
norm_key, asset["roles"] = product.driver.guess_asset_key_and_roles(
|
|
1094
|
+
asset.get("href", ""), product
|
|
1095
|
+
)
|
|
1096
|
+
if norm_key:
|
|
1097
|
+
product.assets[norm_key] = asset
|
|
1098
|
+
# sort assets
|
|
1099
|
+
product.assets.data = dict(sorted(product.assets.data.items()))
|
|
1096
1100
|
products.append(product)
|
|
1097
1101
|
return products
|
|
1098
1102
|
|
|
@@ -1134,7 +1138,7 @@ class QueryStringSearch(Search):
|
|
|
1134
1138
|
total_results = int(count_results)
|
|
1135
1139
|
return total_results
|
|
1136
1140
|
|
|
1137
|
-
def get_collections(self, prep: PreparedSearch, **kwargs: Any) ->
|
|
1141
|
+
def get_collections(self, prep: PreparedSearch, **kwargs: Any) -> tuple[str, ...]:
|
|
1138
1142
|
"""Get the collection to which the product belongs"""
|
|
1139
1143
|
# See https://earth.esa.int/web/sentinel/missions/sentinel-2/news/-
|
|
1140
1144
|
# /asset_publisher/Ac0d/content/change-of
|
|
@@ -1145,7 +1149,7 @@ class QueryStringSearch(Search):
|
|
|
1145
1149
|
not hasattr(prep, "product_type_def_params")
|
|
1146
1150
|
or not prep.product_type_def_params
|
|
1147
1151
|
):
|
|
1148
|
-
collections:
|
|
1152
|
+
collections: set[str] = set()
|
|
1149
1153
|
collection = getattr(self.config, "collection", None)
|
|
1150
1154
|
if collection is None:
|
|
1151
1155
|
try:
|
|
@@ -1187,7 +1191,7 @@ class QueryStringSearch(Search):
|
|
|
1187
1191
|
info_message = prep.info_message
|
|
1188
1192
|
exception_message = prep.exception_message
|
|
1189
1193
|
try:
|
|
1190
|
-
timeout = getattr(self.config, "timeout",
|
|
1194
|
+
timeout = getattr(self.config, "timeout", DEFAULT_SEARCH_TIMEOUT)
|
|
1191
1195
|
ssl_verify = getattr(self.config, "ssl_verify", True)
|
|
1192
1196
|
|
|
1193
1197
|
retry_total = getattr(self.config, "retry_total", REQ_RETRY_TOTAL)
|
|
@@ -1200,7 +1204,7 @@ class QueryStringSearch(Search):
|
|
|
1200
1204
|
|
|
1201
1205
|
ssl_ctx = get_ssl_context(ssl_verify)
|
|
1202
1206
|
# auth if needed
|
|
1203
|
-
kwargs:
|
|
1207
|
+
kwargs: dict[str, Any] = {}
|
|
1204
1208
|
if (
|
|
1205
1209
|
getattr(self.config, "need_auth", False)
|
|
1206
1210
|
and hasattr(prep, "auth")
|
|
@@ -1331,7 +1335,7 @@ class ODataV4Search(QueryStringSearch):
|
|
|
1331
1335
|
|
|
1332
1336
|
def do_search(
|
|
1333
1337
|
self, prep: PreparedSearch = PreparedSearch(), **kwargs: Any
|
|
1334
|
-
) ->
|
|
1338
|
+
) -> list[Any]:
|
|
1335
1339
|
"""A two step search can be performed if the metadata are not given into the search result"""
|
|
1336
1340
|
|
|
1337
1341
|
if getattr(self.config, "per_product_metadata_query", False):
|
|
@@ -1366,7 +1370,7 @@ class ODataV4Search(QueryStringSearch):
|
|
|
1366
1370
|
else:
|
|
1367
1371
|
return super(ODataV4Search, self).do_search(prep, **kwargs)
|
|
1368
1372
|
|
|
1369
|
-
def get_metadata_search_url(self, entity:
|
|
1373
|
+
def get_metadata_search_url(self, entity: dict[str, Any]) -> str:
|
|
1370
1374
|
"""Build the metadata link for the given entity"""
|
|
1371
1375
|
return "{}({})/Metadata".format(
|
|
1372
1376
|
self.config.api_endpoint.rstrip("/"), entity["id"]
|
|
@@ -1374,7 +1378,7 @@ class ODataV4Search(QueryStringSearch):
|
|
|
1374
1378
|
|
|
1375
1379
|
def normalize_results(
|
|
1376
1380
|
self, results: RawSearchResult, **kwargs: Any
|
|
1377
|
-
) ->
|
|
1381
|
+
) -> list[EOProduct]:
|
|
1378
1382
|
"""Build EOProducts from provider results
|
|
1379
1383
|
|
|
1380
1384
|
If configured, a metadata pre-mapping can be applied to simplify further metadata extraction.
|
|
@@ -1431,7 +1435,7 @@ class PostJsonSearch(QueryStringSearch):
|
|
|
1431
1435
|
"""
|
|
1432
1436
|
|
|
1433
1437
|
def _get_default_end_date_from_start_date(
|
|
1434
|
-
self, start_datetime: str, product_type_conf:
|
|
1438
|
+
self, start_datetime: str, product_type_conf: dict[str, Any]
|
|
1435
1439
|
) -> str:
|
|
1436
1440
|
try:
|
|
1437
1441
|
start_date = datetime.fromisoformat(start_datetime)
|
|
@@ -1449,7 +1453,7 @@ class PostJsonSearch(QueryStringSearch):
|
|
|
1449
1453
|
return self.get_product_type_cfg_value("missionEndDate", today().isoformat())
|
|
1450
1454
|
|
|
1451
1455
|
def _check_date_params(
|
|
1452
|
-
self, keywords:
|
|
1456
|
+
self, keywords: dict[str, Any], product_type: Optional[str]
|
|
1453
1457
|
) -> None:
|
|
1454
1458
|
"""checks if start and end date are present in the keywords and adds them if not"""
|
|
1455
1459
|
if (
|
|
@@ -1510,7 +1514,7 @@ class PostJsonSearch(QueryStringSearch):
|
|
|
1510
1514
|
self,
|
|
1511
1515
|
prep: PreparedSearch = PreparedSearch(),
|
|
1512
1516
|
**kwargs: Any,
|
|
1513
|
-
) ->
|
|
1517
|
+
) -> tuple[list[EOProduct], Optional[int]]:
|
|
1514
1518
|
"""Perform a search on an OpenSearch-like interface"""
|
|
1515
1519
|
product_type = kwargs.get("productType", "")
|
|
1516
1520
|
count = prep.count
|
|
@@ -1641,7 +1645,7 @@ class PostJsonSearch(QueryStringSearch):
|
|
|
1641
1645
|
|
|
1642
1646
|
def normalize_results(
|
|
1643
1647
|
self, results: RawSearchResult, **kwargs: Any
|
|
1644
|
-
) ->
|
|
1648
|
+
) -> list[EOProduct]:
|
|
1645
1649
|
"""Build EOProducts from provider results"""
|
|
1646
1650
|
normalized = super().normalize_results(results, **kwargs)
|
|
1647
1651
|
for product in normalized:
|
|
@@ -1676,12 +1680,12 @@ class PostJsonSearch(QueryStringSearch):
|
|
|
1676
1680
|
self,
|
|
1677
1681
|
prep: PreparedSearch = PreparedSearch(),
|
|
1678
1682
|
**kwargs: Any,
|
|
1679
|
-
) ->
|
|
1683
|
+
) -> tuple[list[str], Optional[int]]:
|
|
1680
1684
|
"""Adds pagination to query parameters, and auth to url"""
|
|
1681
1685
|
page = prep.page
|
|
1682
1686
|
items_per_page = prep.items_per_page
|
|
1683
1687
|
count = prep.count
|
|
1684
|
-
urls:
|
|
1688
|
+
urls: list[str] = []
|
|
1685
1689
|
total_results = 0 if count else None
|
|
1686
1690
|
|
|
1687
1691
|
if "count_endpoint" not in self.config.pagination:
|
|
@@ -1750,7 +1754,7 @@ class PostJsonSearch(QueryStringSearch):
|
|
|
1750
1754
|
raise ValidationError("Cannot request empty URL")
|
|
1751
1755
|
info_message = prep.info_message
|
|
1752
1756
|
exception_message = prep.exception_message
|
|
1753
|
-
timeout = getattr(self.config, "timeout",
|
|
1757
|
+
timeout = getattr(self.config, "timeout", DEFAULT_SEARCH_TIMEOUT)
|
|
1754
1758
|
ssl_verify = getattr(self.config, "ssl_verify", True)
|
|
1755
1759
|
try:
|
|
1756
1760
|
# auth if needed
|
|
@@ -1842,7 +1846,7 @@ class StacSearch(PostJsonSearch):
|
|
|
1842
1846
|
|
|
1843
1847
|
def build_query_string(
|
|
1844
1848
|
self, product_type: str, **kwargs: Any
|
|
1845
|
-
) ->
|
|
1849
|
+
) -> tuple[dict[str, Any], str]:
|
|
1846
1850
|
"""Build The query string using the search parameters"""
|
|
1847
1851
|
logger.debug("Building the query string that will be used for search")
|
|
1848
1852
|
|
|
@@ -1868,7 +1872,7 @@ class StacSearch(PostJsonSearch):
|
|
|
1868
1872
|
|
|
1869
1873
|
def discover_queryables(
|
|
1870
1874
|
self, **kwargs: Any
|
|
1871
|
-
) -> Optional[
|
|
1875
|
+
) -> Optional[dict[str, Annotated[Any, FieldInfo]]]:
|
|
1872
1876
|
"""Fetch queryables list from provider using `discover_queryables` conf
|
|
1873
1877
|
|
|
1874
1878
|
:param kwargs: additional filters for queryables (`productType` and other search
|
|
@@ -1964,7 +1968,7 @@ class StacSearch(PostJsonSearch):
|
|
|
1964
1968
|
return None
|
|
1965
1969
|
|
|
1966
1970
|
# convert json results to pydantic model fields
|
|
1967
|
-
field_definitions:
|
|
1971
|
+
field_definitions: dict[str, Any] = dict()
|
|
1968
1972
|
for json_param, json_mtd in json_queryables.items():
|
|
1969
1973
|
param = (
|
|
1970
1974
|
get_queryable_from_provider(
|
|
@@ -1998,6 +2002,6 @@ class PostJsonSearchWithStacQueryables(StacSearch, PostJsonSearch):
|
|
|
1998
2002
|
|
|
1999
2003
|
def build_query_string(
|
|
2000
2004
|
self, product_type: str, **kwargs: Any
|
|
2001
|
-
) ->
|
|
2005
|
+
) -> tuple[dict[str, Any], str]:
|
|
2002
2006
|
"""Build The query string using the search parameters"""
|
|
2003
2007
|
return PostJsonSearch.build_query_string(self, product_type, **kwargs)
|
|
@@ -0,0 +1,85 @@
|
|
|
1
|
+
# -*- coding: utf-8 -*-
|
|
2
|
+
# Copyright 2024, CS GROUP - France, https://www.csgroup.eu/
|
|
3
|
+
#
|
|
4
|
+
# This file is part of EODAG project
|
|
5
|
+
# https://www.github.com/CS-SI/EODAG
|
|
6
|
+
#
|
|
7
|
+
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
8
|
+
# you may not use this file except in compliance with the License.
|
|
9
|
+
# You may obtain a copy of the License at
|
|
10
|
+
#
|
|
11
|
+
# http://www.apache.org/licenses/LICENSE-2.0
|
|
12
|
+
#
|
|
13
|
+
# Unless required by applicable law or agreed to in writing, software
|
|
14
|
+
# distributed under the License is distributed on an "AS IS" BASIS,
|
|
15
|
+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
16
|
+
# See the License for the specific language governing permissions and
|
|
17
|
+
# limitations under the License.
|
|
18
|
+
import logging
|
|
19
|
+
from types import MethodType
|
|
20
|
+
from typing import Any, List
|
|
21
|
+
|
|
22
|
+
from botocore.exceptions import BotoCoreError
|
|
23
|
+
|
|
24
|
+
from eodag.api.product import EOProduct # type: ignore
|
|
25
|
+
from eodag.api.search_result import RawSearchResult
|
|
26
|
+
from eodag.plugins.search.qssearch import StacSearch
|
|
27
|
+
from eodag.utils.exceptions import RequestError
|
|
28
|
+
from eodag.utils.s3 import update_assets_from_s3
|
|
29
|
+
|
|
30
|
+
logger = logging.getLogger("eodag.search.stac_list_assets")
|
|
31
|
+
|
|
32
|
+
|
|
33
|
+
def patched_register_downloader(self, downloader, authenticator):
|
|
34
|
+
"""Add the download information to the product.
|
|
35
|
+
|
|
36
|
+
:param self: product to which information should be added
|
|
37
|
+
:param downloader: The download method that it can use
|
|
38
|
+
:class:`~eodag.plugins.download.base.Download` or
|
|
39
|
+
:class:`~eodag.plugins.api.base.Api`
|
|
40
|
+
:param authenticator: The authentication method needed to perform the download
|
|
41
|
+
:class:`~eodag.plugins.authentication.base.Authentication`
|
|
42
|
+
"""
|
|
43
|
+
# register downloader
|
|
44
|
+
self.register_downloader_only(downloader, authenticator)
|
|
45
|
+
# and also update assets
|
|
46
|
+
try:
|
|
47
|
+
update_assets_from_s3(
|
|
48
|
+
self, authenticator, getattr(downloader.config, "s3_endpoint", None)
|
|
49
|
+
)
|
|
50
|
+
except BotoCoreError as e:
|
|
51
|
+
raise RequestError.from_error(e, "could not update assets") from e
|
|
52
|
+
|
|
53
|
+
|
|
54
|
+
class StacListAssets(StacSearch):
|
|
55
|
+
"""``StacListAssets`` is an extension of :class:`~eodag.plugins.search.qssearch.StacSearch`.
|
|
56
|
+
|
|
57
|
+
It executes a Search on given STAC API endpoint and updates assets with content listed by the plugin using
|
|
58
|
+
``downloadLink`` :class:`~eodag.api.product._product.EOProduct` property.
|
|
59
|
+
|
|
60
|
+
:param provider: provider name
|
|
61
|
+
:param config: It has the same Search plugin configuration as :class:`~eodag.plugins.search.qssearch.StacSearch` and
|
|
62
|
+
one additional parameter:
|
|
63
|
+
|
|
64
|
+
* :attr:`~eodag.config.PluginConfig.s3_endpoint` (``str``): s3 endpoint if not hosted on AWS
|
|
65
|
+
"""
|
|
66
|
+
|
|
67
|
+
def __init__(self, provider, config):
|
|
68
|
+
super(StacSearch, self).__init__(provider, config)
|
|
69
|
+
|
|
70
|
+
def normalize_results(
|
|
71
|
+
self, results: RawSearchResult, **kwargs: Any
|
|
72
|
+
) -> List[EOProduct]:
|
|
73
|
+
"""Build EOProducts from provider results"""
|
|
74
|
+
|
|
75
|
+
products = super(StacSearch, self).normalize_results(results, **kwargs)
|
|
76
|
+
|
|
77
|
+
for product in products:
|
|
78
|
+
# backup original register_downloader to register_downloader_only
|
|
79
|
+
product.register_downloader_only = product.register_downloader
|
|
80
|
+
# patched register_downloader that will also update assets
|
|
81
|
+
product.register_downloader = MethodType(
|
|
82
|
+
patched_register_downloader, product
|
|
83
|
+
)
|
|
84
|
+
|
|
85
|
+
return products
|
|
@@ -18,7 +18,7 @@
|
|
|
18
18
|
from __future__ import annotations
|
|
19
19
|
|
|
20
20
|
import logging
|
|
21
|
-
from typing import TYPE_CHECKING, Any,
|
|
21
|
+
from typing import TYPE_CHECKING, Any, Optional
|
|
22
22
|
from unittest import mock
|
|
23
23
|
|
|
24
24
|
import geojson
|
|
@@ -90,7 +90,7 @@ class StaticStacSearch(StacSearch):
|
|
|
90
90
|
):
|
|
91
91
|
self.config.discover_product_types = {}
|
|
92
92
|
|
|
93
|
-
def discover_product_types(self, **kwargs: Any) -> Optional[
|
|
93
|
+
def discover_product_types(self, **kwargs: Any) -> Optional[dict[str, Any]]:
|
|
94
94
|
"""Fetch product types list from a static STAC Catalog provider using `discover_product_types` conf
|
|
95
95
|
|
|
96
96
|
:returns: configuration dict containing fetched product types information
|
|
@@ -127,7 +127,7 @@ class StaticStacSearch(StacSearch):
|
|
|
127
127
|
self,
|
|
128
128
|
prep: PreparedSearch = PreparedSearch(),
|
|
129
129
|
**kwargs: Any,
|
|
130
|
-
) ->
|
|
130
|
+
) -> tuple[list[EOProduct], Optional[int]]:
|
|
131
131
|
"""Perform a search on a static STAC Catalog"""
|
|
132
132
|
|
|
133
133
|
# only return 1 page if pagination is disabled
|