eodag 3.1.0b1__py3-none-any.whl → 3.2.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- eodag/api/core.py +69 -63
- eodag/api/product/_assets.py +49 -13
- eodag/api/product/_product.py +41 -30
- eodag/api/product/drivers/__init__.py +81 -4
- eodag/api/product/drivers/base.py +65 -4
- eodag/api/product/drivers/generic.py +65 -0
- eodag/api/product/drivers/sentinel1.py +97 -0
- eodag/api/product/drivers/sentinel2.py +95 -0
- eodag/api/product/metadata_mapping.py +85 -79
- eodag/api/search_result.py +13 -23
- eodag/cli.py +4 -4
- eodag/config.py +77 -80
- eodag/plugins/apis/base.py +1 -1
- eodag/plugins/apis/ecmwf.py +12 -15
- eodag/plugins/apis/usgs.py +12 -11
- eodag/plugins/authentication/aws_auth.py +16 -13
- eodag/plugins/authentication/base.py +5 -3
- eodag/plugins/authentication/header.py +3 -3
- eodag/plugins/authentication/keycloak.py +4 -4
- eodag/plugins/authentication/oauth.py +7 -3
- eodag/plugins/authentication/openid_connect.py +20 -14
- eodag/plugins/authentication/sas_auth.py +4 -4
- eodag/plugins/authentication/token.py +7 -7
- eodag/plugins/authentication/token_exchange.py +1 -1
- eodag/plugins/base.py +4 -4
- eodag/plugins/crunch/base.py +4 -4
- eodag/plugins/crunch/filter_date.py +4 -4
- eodag/plugins/crunch/filter_latest_intersect.py +6 -6
- eodag/plugins/crunch/filter_latest_tpl_name.py +7 -7
- eodag/plugins/crunch/filter_overlap.py +4 -4
- eodag/plugins/crunch/filter_property.py +4 -4
- eodag/plugins/download/aws.py +137 -77
- eodag/plugins/download/base.py +8 -17
- eodag/plugins/download/creodias_s3.py +2 -2
- eodag/plugins/download/http.py +30 -32
- eodag/plugins/download/s3rest.py +5 -4
- eodag/plugins/manager.py +10 -20
- eodag/plugins/search/__init__.py +6 -5
- eodag/plugins/search/base.py +38 -42
- eodag/plugins/search/build_search_result.py +286 -336
- eodag/plugins/search/cop_marine.py +22 -12
- eodag/plugins/search/creodias_s3.py +8 -78
- eodag/plugins/search/csw.py +11 -11
- eodag/plugins/search/data_request_search.py +19 -18
- eodag/plugins/search/qssearch.py +84 -151
- eodag/plugins/search/stac_list_assets.py +85 -0
- eodag/plugins/search/static_stac_search.py +4 -4
- eodag/resources/ext_product_types.json +1 -1
- eodag/resources/product_types.yml +848 -398
- eodag/resources/providers.yml +1038 -1115
- eodag/resources/stac_api.yml +2 -2
- eodag/resources/user_conf_template.yml +10 -9
- eodag/rest/cache.py +2 -2
- eodag/rest/config.py +3 -3
- eodag/rest/core.py +24 -24
- eodag/rest/errors.py +5 -5
- eodag/rest/server.py +3 -11
- eodag/rest/stac.py +41 -38
- eodag/rest/types/collections_search.py +3 -3
- eodag/rest/types/eodag_search.py +23 -23
- eodag/rest/types/queryables.py +40 -28
- eodag/rest/types/stac_search.py +15 -25
- eodag/rest/utils/__init__.py +11 -21
- eodag/rest/utils/cql_evaluate.py +6 -6
- eodag/rest/utils/rfc3339.py +2 -2
- eodag/types/__init__.py +97 -29
- eodag/types/bbox.py +2 -2
- eodag/types/download_args.py +2 -2
- eodag/types/queryables.py +5 -2
- eodag/types/search_args.py +4 -4
- eodag/types/whoosh.py +1 -3
- eodag/utils/__init__.py +82 -41
- eodag/utils/exceptions.py +2 -2
- eodag/utils/import_system.py +2 -2
- eodag/utils/requests.py +2 -2
- eodag/utils/rest.py +2 -2
- eodag/utils/s3.py +231 -0
- eodag/utils/stac_reader.py +10 -10
- {eodag-3.1.0b1.dist-info → eodag-3.2.0.dist-info}/METADATA +12 -10
- eodag-3.2.0.dist-info/RECORD +113 -0
- {eodag-3.1.0b1.dist-info → eodag-3.2.0.dist-info}/WHEEL +1 -1
- {eodag-3.1.0b1.dist-info → eodag-3.2.0.dist-info}/entry_points.txt +1 -0
- eodag-3.1.0b1.dist-info/RECORD +0 -108
- {eodag-3.1.0b1.dist-info → eodag-3.2.0.dist-info/licenses}/LICENSE +0 -0
- {eodag-3.1.0b1.dist-info → eodag-3.2.0.dist-info}/top_level.txt +0 -0
eodag/plugins/search/qssearch.py
CHANGED
|
@@ -19,19 +19,15 @@ from __future__ import annotations
|
|
|
19
19
|
|
|
20
20
|
import logging
|
|
21
21
|
import re
|
|
22
|
+
import socket
|
|
22
23
|
from copy import copy as copy_copy
|
|
23
|
-
from datetime import datetime, timedelta
|
|
24
24
|
from typing import (
|
|
25
25
|
TYPE_CHECKING,
|
|
26
26
|
Annotated,
|
|
27
27
|
Any,
|
|
28
28
|
Callable,
|
|
29
|
-
Dict,
|
|
30
|
-
List,
|
|
31
29
|
Optional,
|
|
32
30
|
Sequence,
|
|
33
|
-
Set,
|
|
34
|
-
Tuple,
|
|
35
31
|
TypedDict,
|
|
36
32
|
cast,
|
|
37
33
|
get_args,
|
|
@@ -52,7 +48,6 @@ import geojson
|
|
|
52
48
|
import orjson
|
|
53
49
|
import requests
|
|
54
50
|
import yaml
|
|
55
|
-
from dateutil.utils import today
|
|
56
51
|
from jsonpath_ng import JSONPath
|
|
57
52
|
from lxml import etree
|
|
58
53
|
from pydantic import create_model
|
|
@@ -77,7 +72,7 @@ from eodag.plugins.search.base import Search
|
|
|
77
72
|
from eodag.types import json_field_definition_to_python, model_fields_to_annotated
|
|
78
73
|
from eodag.types.search_args import SortByList
|
|
79
74
|
from eodag.utils import (
|
|
80
|
-
|
|
75
|
+
DEFAULT_SEARCH_TIMEOUT,
|
|
81
76
|
GENERIC_PRODUCT_TYPE,
|
|
82
77
|
HTTP_REQ_TIMEOUT,
|
|
83
78
|
REQ_RETRY_BACKOFF_FACTOR,
|
|
@@ -128,7 +123,9 @@ class QueryStringSearch(Search):
|
|
|
128
123
|
authentication error; only used if ``need_auth=true``
|
|
129
124
|
* :attr:`~eodag.config.PluginConfig.ssl_verify` (``bool``): if the ssl certificates should be verified in
|
|
130
125
|
requests; default: ``True``
|
|
131
|
-
* :attr:`~eodag.config.PluginConfig.
|
|
126
|
+
* :attr:`~eodag.config.PluginConfig.asset_key_from_href` (``bool``): guess assets keys using their ``href``. Use
|
|
127
|
+
their original key if ``False``; default: ``True``
|
|
128
|
+
* :attr:`~eodag.config.PluginConfig.dont_quote` (``list[str]``): characters that should not be quoted in the
|
|
132
129
|
url params
|
|
133
130
|
* :attr:`~eodag.config.PluginConfig.timeout` (``int``): time to wait until request timeout in seconds;
|
|
134
131
|
default: ``5``
|
|
@@ -136,10 +133,10 @@ class QueryStringSearch(Search):
|
|
|
136
133
|
total number of retries to allow; default: ``3``
|
|
137
134
|
* :attr:`~eodag.config.PluginConfig.retry_backoff_factor` (``int``): :class:`urllib3.util.Retry`
|
|
138
135
|
``backoff_factor`` parameter, backoff factor to apply between attempts after the second try; default: ``2``
|
|
139
|
-
* :attr:`~eodag.config.PluginConfig.retry_status_forcelist` (``
|
|
136
|
+
* :attr:`~eodag.config.PluginConfig.retry_status_forcelist` (``list[int]``): :class:`urllib3.util.Retry`
|
|
140
137
|
``status_forcelist`` parameter, list of integer HTTP status codes that we should force a retry on; default:
|
|
141
138
|
``[401, 429, 500, 502, 503, 504]``
|
|
142
|
-
* :attr:`~eodag.config.PluginConfig.literal_search_params` (``
|
|
139
|
+
* :attr:`~eodag.config.PluginConfig.literal_search_params` (``dict[str, str]``): A mapping of (search_param =>
|
|
143
140
|
search_value) pairs giving search parameters to be passed as is in the search url query string. This is useful
|
|
144
141
|
for example in situations where the user wants to add a fixed search query parameter exactly
|
|
145
142
|
as it is done on the provider interface.
|
|
@@ -183,13 +180,13 @@ class QueryStringSearch(Search):
|
|
|
183
180
|
* :attr:`~eodag.config.PluginConfig.DiscoverProductTypes.generic_product_type_id` (``str``): mapping for the
|
|
184
181
|
product type id
|
|
185
182
|
* :attr:`~eodag.config.PluginConfig.DiscoverProductTypes.generic_product_type_parsable_metadata`
|
|
186
|
-
(``
|
|
183
|
+
(``dict[str, str]``): mapping for product type metadata (e.g. ``abstract``, ``licence``) which can be parsed
|
|
187
184
|
from the provider result
|
|
188
185
|
* :attr:`~eodag.config.PluginConfig.DiscoverProductTypes.generic_product_type_parsable_properties`
|
|
189
|
-
(``
|
|
186
|
+
(``dict[str, str]``): mapping for product type properties which can be parsed from the result and are not
|
|
190
187
|
product type metadata
|
|
191
188
|
* :attr:`~eodag.config.PluginConfig.DiscoverProductTypes.generic_product_type_unparsable_properties`
|
|
192
|
-
(``
|
|
189
|
+
(``dict[str, str]``): mapping for product type properties which cannot be parsed from the result and are not
|
|
193
190
|
product type metadata
|
|
194
191
|
* :attr:`~eodag.config.PluginConfig.DiscoverProductTypes.single_collection_fetch_url` (``str``): url to fetch
|
|
195
192
|
data for a single collection; used if product type metadata is not available from the endpoint given in
|
|
@@ -198,13 +195,13 @@ class QueryStringSearch(Search):
|
|
|
198
195
|
to be added to the :attr:`~eodag.config.PluginConfig.DiscoverProductTypes.fetch_url` to filter for a
|
|
199
196
|
collection
|
|
200
197
|
* :attr:`~eodag.config.PluginConfig.DiscoverProductTypes.single_product_type_parsable_metadata`
|
|
201
|
-
(``
|
|
198
|
+
(``dict[str, str]``): mapping for product type metadata returned by the endpoint given in
|
|
202
199
|
:attr:`~eodag.config.PluginConfig.DiscoverProductTypes.single_collection_fetch_url`.
|
|
203
200
|
|
|
204
201
|
* :attr:`~eodag.config.PluginConfig.sort` (:class:`~eodag.config.PluginConfig.Sort`): configuration for sorting
|
|
205
202
|
the results. It contains the keys:
|
|
206
203
|
|
|
207
|
-
* :attr:`~eodag.config.PluginConfig.Sort.sort_by_default` (``
|
|
204
|
+
* :attr:`~eodag.config.PluginConfig.Sort.sort_by_default` (``list[Tuple(str, Literal["ASC", "DESC"])]``):
|
|
208
205
|
parameter and sort order by which the result will be sorted by default (if the user does not enter a
|
|
209
206
|
``sort_by`` parameter); if not given the result will use the default sorting of the provider; Attention:
|
|
210
207
|
for some providers sorting might cause a timeout if no filters are used. In that case no default
|
|
@@ -220,12 +217,12 @@ class QueryStringSearch(Search):
|
|
|
220
217
|
* :attr:`~eodag.config.PluginConfig.Sort.sort_param_mapping` (``Dict [str, str]``): mapping for the parameters
|
|
221
218
|
available for sorting
|
|
222
219
|
* :attr:`~eodag.config.PluginConfig.Sort.sort_order_mapping`
|
|
223
|
-
(``
|
|
220
|
+
(``dict[Literal["ascending", "descending"], str]``): mapping for the sort order
|
|
224
221
|
* :attr:`~eodag.config.PluginConfig.Sort.max_sort_params` (``int``): maximum number of sort parameters
|
|
225
222
|
supported by the provider; used to validate the user input to avoid failed requests or unexpected behaviour
|
|
226
223
|
(not all parameters are used in the request)
|
|
227
224
|
|
|
228
|
-
* :attr:`~eodag.config.PluginConfig.metadata_mapping` (``
|
|
225
|
+
* :attr:`~eodag.config.PluginConfig.metadata_mapping` (``dict[str, Any]``): The search plugins of this kind can
|
|
229
226
|
detect when a metadata mapping is "query-able", and get the semantics of how to format the query string
|
|
230
227
|
parameter that enables to make a query on the corresponding metadata. To make a metadata query-able,
|
|
231
228
|
just configure it in the metadata mapping to be a list of 2 items, the first one being the
|
|
@@ -258,7 +255,7 @@ class QueryStringSearch(Search):
|
|
|
258
255
|
metadata is activated; default: ``False``; if false, the other parameters are not used;
|
|
259
256
|
* :attr:`~eodag.config.PluginConfig.DiscoverMetadata.metadata_pattern` (``str``): regex string a parameter in
|
|
260
257
|
the result should match so that is used
|
|
261
|
-
* :attr:`~eodag.config.PluginConfig.DiscoverMetadata.search_param` (``Union [str,
|
|
258
|
+
* :attr:`~eodag.config.PluginConfig.DiscoverMetadata.search_param` (``Union [str, dict[str, Any]]``): format
|
|
262
259
|
to add a query param given by the user and not in the metadata mapping to the requests, 'metadata' will be
|
|
263
260
|
replaced by the search param; can be a string or a dict containing
|
|
264
261
|
:attr:`~eodag.config.PluginConfig.free_text_search_operations`
|
|
@@ -286,7 +283,7 @@ class QueryStringSearch(Search):
|
|
|
286
283
|
the result is an array of constraints
|
|
287
284
|
"""
|
|
288
285
|
|
|
289
|
-
extract_properties:
|
|
286
|
+
extract_properties: dict[str, Callable[..., dict[str, Any]]] = {
|
|
290
287
|
"xml": properties_from_xml,
|
|
291
288
|
"json": properties_from_json,
|
|
292
289
|
}
|
|
@@ -297,8 +294,8 @@ class QueryStringSearch(Search):
|
|
|
297
294
|
self.config.__dict__.setdefault("results_entry", "features")
|
|
298
295
|
self.config.__dict__.setdefault("pagination", {})
|
|
299
296
|
self.config.__dict__.setdefault("free_text_search_operations", {})
|
|
300
|
-
self.search_urls:
|
|
301
|
-
self.query_params:
|
|
297
|
+
self.search_urls: list[str] = []
|
|
298
|
+
self.query_params: dict[str, str] = dict()
|
|
302
299
|
self.query_string = ""
|
|
303
300
|
self.next_page_url = None
|
|
304
301
|
self.next_page_query_obj = None
|
|
@@ -443,7 +440,7 @@ class QueryStringSearch(Search):
|
|
|
443
440
|
self.next_page_query_obj = None
|
|
444
441
|
self.next_page_merge = None
|
|
445
442
|
|
|
446
|
-
def discover_product_types(self, **kwargs: Any) -> Optional[
|
|
443
|
+
def discover_product_types(self, **kwargs: Any) -> Optional[dict[str, Any]]:
|
|
447
444
|
"""Fetch product types list from provider using `discover_product_types` conf
|
|
448
445
|
|
|
449
446
|
:returns: configuration dict containing fetched product types information
|
|
@@ -460,7 +457,7 @@ class QueryStringSearch(Search):
|
|
|
460
457
|
# no pagination
|
|
461
458
|
return self.discover_product_types_per_page(**kwargs)
|
|
462
459
|
|
|
463
|
-
conf_update_dict:
|
|
460
|
+
conf_update_dict: dict[str, Any] = {
|
|
464
461
|
"providers_config": {},
|
|
465
462
|
"product_types_config": {},
|
|
466
463
|
}
|
|
@@ -493,7 +490,7 @@ class QueryStringSearch(Search):
|
|
|
493
490
|
|
|
494
491
|
def discover_product_types_per_page(
|
|
495
492
|
self, **kwargs: Any
|
|
496
|
-
) -> Optional[
|
|
493
|
+
) -> Optional[dict[str, Any]]:
|
|
497
494
|
"""Fetch product types list from provider using `discover_product_types` conf
|
|
498
495
|
using paginated ``kwargs["fetch_url"]``
|
|
499
496
|
|
|
@@ -536,7 +533,7 @@ class QueryStringSearch(Search):
|
|
|
536
533
|
|
|
537
534
|
prep.info_message = "Fetching product types: {}".format(prep.url)
|
|
538
535
|
prep.exception_message = (
|
|
539
|
-
"Skipping error while fetching product types for
|
|
536
|
+
"Skipping error while fetching product types for {} {} instance:"
|
|
540
537
|
).format(self.provider, self.__class__.__name__)
|
|
541
538
|
|
|
542
539
|
# Query using appropriate method
|
|
@@ -551,7 +548,7 @@ class QueryStringSearch(Search):
|
|
|
551
548
|
return None
|
|
552
549
|
else:
|
|
553
550
|
try:
|
|
554
|
-
conf_update_dict:
|
|
551
|
+
conf_update_dict: dict[str, Any] = {
|
|
555
552
|
"providers_config": {},
|
|
556
553
|
"product_types_config": {},
|
|
557
554
|
}
|
|
@@ -570,7 +567,7 @@ class QueryStringSearch(Search):
|
|
|
570
567
|
result = result[0]
|
|
571
568
|
|
|
572
569
|
def conf_update_from_product_type_result(
|
|
573
|
-
product_type_result:
|
|
570
|
+
product_type_result: dict[str, Any],
|
|
574
571
|
) -> None:
|
|
575
572
|
"""Update ``conf_update_dict`` using given product type json response"""
|
|
576
573
|
# providers_config extraction
|
|
@@ -698,7 +695,7 @@ class QueryStringSearch(Search):
|
|
|
698
695
|
|
|
699
696
|
def _get_product_type_metadata_from_single_collection_endpoint(
|
|
700
697
|
self, product_type: str
|
|
701
|
-
) ->
|
|
698
|
+
) -> dict[str, Any]:
|
|
702
699
|
"""
|
|
703
700
|
retrieves additional product type information from an endpoint returning data for a single collection
|
|
704
701
|
:param product_type: product type
|
|
@@ -726,7 +723,7 @@ class QueryStringSearch(Search):
|
|
|
726
723
|
self,
|
|
727
724
|
prep: PreparedSearch = PreparedSearch(),
|
|
728
725
|
**kwargs: Any,
|
|
729
|
-
) ->
|
|
726
|
+
) -> tuple[list[EOProduct], Optional[int]]:
|
|
730
727
|
"""Perform a search on an OpenSearch-like interface
|
|
731
728
|
|
|
732
729
|
:param prep: Object collecting needed information for search.
|
|
@@ -754,7 +751,7 @@ class QueryStringSearch(Search):
|
|
|
754
751
|
|
|
755
752
|
# provider product type specific conf
|
|
756
753
|
prep.product_type_def_params = (
|
|
757
|
-
self.get_product_type_def_params(product_type,
|
|
754
|
+
self.get_product_type_def_params(product_type, format_variables=kwargs)
|
|
758
755
|
if product_type is not None
|
|
759
756
|
else {}
|
|
760
757
|
)
|
|
@@ -778,7 +775,7 @@ class QueryStringSearch(Search):
|
|
|
778
775
|
}
|
|
779
776
|
)
|
|
780
777
|
|
|
781
|
-
qp, qs = self.build_query_string(product_type,
|
|
778
|
+
qp, qs = self.build_query_string(product_type, keywords)
|
|
782
779
|
|
|
783
780
|
prep.query_params = qp
|
|
784
781
|
prep.query_string = qs
|
|
@@ -806,21 +803,21 @@ class QueryStringSearch(Search):
|
|
|
806
803
|
reason="Simply run `self.config.metadata_mapping.update(metadata_mapping)` instead",
|
|
807
804
|
version="2.10.0",
|
|
808
805
|
)
|
|
809
|
-
def update_metadata_mapping(self, metadata_mapping:
|
|
806
|
+
def update_metadata_mapping(self, metadata_mapping: dict[str, Any]) -> None:
|
|
810
807
|
"""Update plugin metadata_mapping with input metadata_mapping configuration"""
|
|
811
808
|
if self.config.metadata_mapping:
|
|
812
809
|
self.config.metadata_mapping.update(metadata_mapping)
|
|
813
810
|
|
|
814
811
|
def build_query_string(
|
|
815
|
-
self, product_type: str,
|
|
816
|
-
) ->
|
|
812
|
+
self, product_type: str, query_dict: dict[str, Any]
|
|
813
|
+
) -> tuple[dict[str, Any], str]:
|
|
817
814
|
"""Build The query string using the search parameters"""
|
|
818
815
|
logger.debug("Building the query string that will be used for search")
|
|
819
|
-
query_params = format_query_params(product_type, self.config,
|
|
816
|
+
query_params = format_query_params(product_type, self.config, query_dict)
|
|
820
817
|
|
|
821
818
|
# Build the final query string, in one go without quoting it
|
|
822
819
|
# (some providers do not operate well with urlencoded and quoted query strings)
|
|
823
|
-
def quote_via(x: Any, *_args, **_kwargs) -> str:
|
|
820
|
+
def quote_via(x: Any, *_args: Any, **_kwargs: Any) -> str:
|
|
824
821
|
return x
|
|
825
822
|
|
|
826
823
|
return (
|
|
@@ -832,7 +829,7 @@ class QueryStringSearch(Search):
|
|
|
832
829
|
self,
|
|
833
830
|
prep: PreparedSearch = PreparedSearch(page=None, items_per_page=None),
|
|
834
831
|
**kwargs: Any,
|
|
835
|
-
) ->
|
|
832
|
+
) -> tuple[list[str], Optional[int]]:
|
|
836
833
|
"""Build paginated urls"""
|
|
837
834
|
page = prep.page
|
|
838
835
|
items_per_page = prep.items_per_page
|
|
@@ -901,7 +898,7 @@ class QueryStringSearch(Search):
|
|
|
901
898
|
|
|
902
899
|
def do_search(
|
|
903
900
|
self, prep: PreparedSearch = PreparedSearch(items_per_page=None), **kwargs: Any
|
|
904
|
-
) ->
|
|
901
|
+
) -> list[Any]:
|
|
905
902
|
"""Perform the actual search request.
|
|
906
903
|
|
|
907
904
|
If there is a specified number of items per page, return the results as soon
|
|
@@ -918,7 +915,7 @@ class QueryStringSearch(Search):
|
|
|
918
915
|
"total_items_nb_key_path"
|
|
919
916
|
]
|
|
920
917
|
|
|
921
|
-
results:
|
|
918
|
+
results: list[Any] = []
|
|
922
919
|
for search_url in prep.search_urls:
|
|
923
920
|
single_search_prep = copy_copy(prep)
|
|
924
921
|
single_search_prep.url = search_url
|
|
@@ -1069,14 +1066,15 @@ class QueryStringSearch(Search):
|
|
|
1069
1066
|
|
|
1070
1067
|
def normalize_results(
|
|
1071
1068
|
self, results: RawSearchResult, **kwargs: Any
|
|
1072
|
-
) ->
|
|
1069
|
+
) -> list[EOProduct]:
|
|
1073
1070
|
"""Build EOProducts from provider results"""
|
|
1074
1071
|
normalize_remaining_count = len(results)
|
|
1075
1072
|
logger.debug(
|
|
1076
1073
|
"Adapting %s plugin results to eodag product representation"
|
|
1077
1074
|
% normalize_remaining_count
|
|
1078
1075
|
)
|
|
1079
|
-
products:
|
|
1076
|
+
products: list[EOProduct] = []
|
|
1077
|
+
asset_key_from_href = getattr(self.config, "asset_key_from_href", True)
|
|
1080
1078
|
for result in results:
|
|
1081
1079
|
product = EOProduct(
|
|
1082
1080
|
self.provider,
|
|
@@ -1091,8 +1089,16 @@ class QueryStringSearch(Search):
|
|
|
1091
1089
|
product.properties = dict(
|
|
1092
1090
|
getattr(self.config, "product_type_config", {}), **product.properties
|
|
1093
1091
|
)
|
|
1094
|
-
# move assets from properties to product's attr
|
|
1095
|
-
product.
|
|
1092
|
+
# move assets from properties to product's attr, normalize keys & roles
|
|
1093
|
+
for key, asset in product.properties.pop("assets", {}).items():
|
|
1094
|
+
norm_key, asset["roles"] = product.driver.guess_asset_key_and_roles(
|
|
1095
|
+
asset.get("href", "") if asset_key_from_href else key,
|
|
1096
|
+
product,
|
|
1097
|
+
)
|
|
1098
|
+
if norm_key:
|
|
1099
|
+
product.assets[norm_key] = asset
|
|
1100
|
+
# sort assets
|
|
1101
|
+
product.assets.data = dict(sorted(product.assets.data.items()))
|
|
1096
1102
|
products.append(product)
|
|
1097
1103
|
return products
|
|
1098
1104
|
|
|
@@ -1134,7 +1140,7 @@ class QueryStringSearch(Search):
|
|
|
1134
1140
|
total_results = int(count_results)
|
|
1135
1141
|
return total_results
|
|
1136
1142
|
|
|
1137
|
-
def get_collections(self, prep: PreparedSearch, **kwargs: Any) ->
|
|
1143
|
+
def get_collections(self, prep: PreparedSearch, **kwargs: Any) -> tuple[str, ...]:
|
|
1138
1144
|
"""Get the collection to which the product belongs"""
|
|
1139
1145
|
# See https://earth.esa.int/web/sentinel/missions/sentinel-2/news/-
|
|
1140
1146
|
# /asset_publisher/Ac0d/content/change-of
|
|
@@ -1145,7 +1151,7 @@ class QueryStringSearch(Search):
|
|
|
1145
1151
|
not hasattr(prep, "product_type_def_params")
|
|
1146
1152
|
or not prep.product_type_def_params
|
|
1147
1153
|
):
|
|
1148
|
-
collections:
|
|
1154
|
+
collections: set[str] = set()
|
|
1149
1155
|
collection = getattr(self.config, "collection", None)
|
|
1150
1156
|
if collection is None:
|
|
1151
1157
|
try:
|
|
@@ -1187,7 +1193,7 @@ class QueryStringSearch(Search):
|
|
|
1187
1193
|
info_message = prep.info_message
|
|
1188
1194
|
exception_message = prep.exception_message
|
|
1189
1195
|
try:
|
|
1190
|
-
timeout = getattr(self.config, "timeout",
|
|
1196
|
+
timeout = getattr(self.config, "timeout", DEFAULT_SEARCH_TIMEOUT)
|
|
1191
1197
|
ssl_verify = getattr(self.config, "ssl_verify", True)
|
|
1192
1198
|
|
|
1193
1199
|
retry_total = getattr(self.config, "retry_total", REQ_RETRY_TOTAL)
|
|
@@ -1200,7 +1206,7 @@ class QueryStringSearch(Search):
|
|
|
1200
1206
|
|
|
1201
1207
|
ssl_ctx = get_ssl_context(ssl_verify)
|
|
1202
1208
|
# auth if needed
|
|
1203
|
-
kwargs:
|
|
1209
|
+
kwargs: dict[str, Any] = {}
|
|
1204
1210
|
if (
|
|
1205
1211
|
getattr(self.config, "need_auth", False)
|
|
1206
1212
|
and hasattr(prep, "auth")
|
|
@@ -1255,6 +1261,9 @@ class QueryStringSearch(Search):
|
|
|
1255
1261
|
response.raise_for_status()
|
|
1256
1262
|
except requests.exceptions.Timeout as exc:
|
|
1257
1263
|
raise TimeOutError(exc, timeout=timeout) from exc
|
|
1264
|
+
except socket.timeout:
|
|
1265
|
+
err = requests.exceptions.Timeout(request=requests.Request(url=url))
|
|
1266
|
+
raise TimeOutError(err, timeout=timeout)
|
|
1258
1267
|
except (requests.RequestException, URLError) as err:
|
|
1259
1268
|
err_msg = err.readlines() if hasattr(err, "readlines") else ""
|
|
1260
1269
|
if exception_message:
|
|
@@ -1331,7 +1340,7 @@ class ODataV4Search(QueryStringSearch):
|
|
|
1331
1340
|
|
|
1332
1341
|
def do_search(
|
|
1333
1342
|
self, prep: PreparedSearch = PreparedSearch(), **kwargs: Any
|
|
1334
|
-
) ->
|
|
1343
|
+
) -> list[Any]:
|
|
1335
1344
|
"""A two step search can be performed if the metadata are not given into the search result"""
|
|
1336
1345
|
|
|
1337
1346
|
if getattr(self.config, "per_product_metadata_query", False):
|
|
@@ -1366,7 +1375,7 @@ class ODataV4Search(QueryStringSearch):
|
|
|
1366
1375
|
else:
|
|
1367
1376
|
return super(ODataV4Search, self).do_search(prep, **kwargs)
|
|
1368
1377
|
|
|
1369
|
-
def get_metadata_search_url(self, entity:
|
|
1378
|
+
def get_metadata_search_url(self, entity: dict[str, Any]) -> str:
|
|
1370
1379
|
"""Build the metadata link for the given entity"""
|
|
1371
1380
|
return "{}({})/Metadata".format(
|
|
1372
1381
|
self.config.api_endpoint.rstrip("/"), entity["id"]
|
|
@@ -1374,7 +1383,7 @@ class ODataV4Search(QueryStringSearch):
|
|
|
1374
1383
|
|
|
1375
1384
|
def normalize_results(
|
|
1376
1385
|
self, results: RawSearchResult, **kwargs: Any
|
|
1377
|
-
) ->
|
|
1386
|
+
) -> list[EOProduct]:
|
|
1378
1387
|
"""Build EOProducts from provider results
|
|
1379
1388
|
|
|
1380
1389
|
If configured, a metadata pre-mapping can be applied to simplify further metadata extraction.
|
|
@@ -1430,87 +1439,11 @@ class PostJsonSearch(QueryStringSearch):
|
|
|
1430
1439
|
|
|
1431
1440
|
"""
|
|
1432
1441
|
|
|
1433
|
-
def _get_default_end_date_from_start_date(
|
|
1434
|
-
self, start_datetime: str, product_type_conf: Dict[str, Any]
|
|
1435
|
-
) -> str:
|
|
1436
|
-
try:
|
|
1437
|
-
start_date = datetime.fromisoformat(start_datetime)
|
|
1438
|
-
except ValueError:
|
|
1439
|
-
start_date = datetime.strptime(start_datetime, "%Y-%m-%dT%H:%M:%SZ")
|
|
1440
|
-
if "completionTimeFromAscendingNode" in product_type_conf:
|
|
1441
|
-
mapping = product_type_conf["completionTimeFromAscendingNode"]
|
|
1442
|
-
# if date is mapped to year/month/(day), use end_date = start_date else start_date + 1 day
|
|
1443
|
-
# (default dates are only needed for ecmwf products where selected timespans should not be too large)
|
|
1444
|
-
if isinstance(mapping, list) and "year" in mapping[0]:
|
|
1445
|
-
end_date = start_date
|
|
1446
|
-
else:
|
|
1447
|
-
end_date = start_date + timedelta(days=1)
|
|
1448
|
-
return end_date.isoformat()
|
|
1449
|
-
return self.get_product_type_cfg_value("missionEndDate", today().isoformat())
|
|
1450
|
-
|
|
1451
|
-
def _check_date_params(
|
|
1452
|
-
self, keywords: Dict[str, Any], product_type: Optional[str]
|
|
1453
|
-
) -> None:
|
|
1454
|
-
"""checks if start and end date are present in the keywords and adds them if not"""
|
|
1455
|
-
if (
|
|
1456
|
-
"startTimeFromAscendingNode"
|
|
1457
|
-
and "completionTimeFromAscendingNode" in keywords
|
|
1458
|
-
):
|
|
1459
|
-
return
|
|
1460
|
-
|
|
1461
|
-
product_type_conf = getattr(self.config, "metadata_mapping", {})
|
|
1462
|
-
if (
|
|
1463
|
-
product_type
|
|
1464
|
-
and product_type in self.config.products
|
|
1465
|
-
and "metadata_mapping" in self.config.products[product_type]
|
|
1466
|
-
):
|
|
1467
|
-
product_type_conf = self.config.products[product_type]["metadata_mapping"]
|
|
1468
|
-
# start time given, end time missing
|
|
1469
|
-
if "startTimeFromAscendingNode" in keywords:
|
|
1470
|
-
keywords[
|
|
1471
|
-
"completionTimeFromAscendingNode"
|
|
1472
|
-
] = self._get_default_end_date_from_start_date(
|
|
1473
|
-
keywords["startTimeFromAscendingNode"], product_type_conf
|
|
1474
|
-
)
|
|
1475
|
-
return
|
|
1476
|
-
|
|
1477
|
-
if "completionTimeFromAscendingNode" in product_type_conf:
|
|
1478
|
-
mapping = product_type_conf["startTimeFromAscendingNode"]
|
|
1479
|
-
if not isinstance(mapping, list):
|
|
1480
|
-
mapping = product_type_conf["completionTimeFromAscendingNode"]
|
|
1481
|
-
if isinstance(mapping, list):
|
|
1482
|
-
# get time parameters (date, year, month, ...) from metadata mapping
|
|
1483
|
-
input_mapping = mapping[0].replace("{{", "").replace("}}", "")
|
|
1484
|
-
time_params = [
|
|
1485
|
-
values.split(":")[0].strip() for values in input_mapping.split(",")
|
|
1486
|
-
]
|
|
1487
|
-
time_params = [
|
|
1488
|
-
tp.replace('"', "").replace("'", "") for tp in time_params
|
|
1489
|
-
]
|
|
1490
|
-
# if startTime is not given but other time params (e.g. year/month/(day)) are given,
|
|
1491
|
-
# no default date is required
|
|
1492
|
-
in_keywords = True
|
|
1493
|
-
for tp in time_params:
|
|
1494
|
-
if tp not in keywords:
|
|
1495
|
-
in_keywords = False
|
|
1496
|
-
break
|
|
1497
|
-
if not in_keywords:
|
|
1498
|
-
keywords[
|
|
1499
|
-
"startTimeFromAscendingNode"
|
|
1500
|
-
] = self.get_product_type_cfg_value(
|
|
1501
|
-
"missionStartDate", DEFAULT_MISSION_START_DATE
|
|
1502
|
-
)
|
|
1503
|
-
keywords[
|
|
1504
|
-
"completionTimeFromAscendingNode"
|
|
1505
|
-
] = self._get_default_end_date_from_start_date(
|
|
1506
|
-
keywords["startTimeFromAscendingNode"], product_type_conf
|
|
1507
|
-
)
|
|
1508
|
-
|
|
1509
1442
|
def query(
|
|
1510
1443
|
self,
|
|
1511
1444
|
prep: PreparedSearch = PreparedSearch(),
|
|
1512
1445
|
**kwargs: Any,
|
|
1513
|
-
) ->
|
|
1446
|
+
) -> tuple[list[EOProduct], Optional[int]]:
|
|
1514
1447
|
"""Perform a search on an OpenSearch-like interface"""
|
|
1515
1448
|
product_type = kwargs.get("productType", "")
|
|
1516
1449
|
count = prep.count
|
|
@@ -1528,7 +1461,7 @@ class PostJsonSearch(QueryStringSearch):
|
|
|
1528
1461
|
|
|
1529
1462
|
# provider product type specific conf
|
|
1530
1463
|
prep.product_type_def_params = self.get_product_type_def_params(
|
|
1531
|
-
product_type,
|
|
1464
|
+
product_type, format_variables=kwargs
|
|
1532
1465
|
)
|
|
1533
1466
|
else:
|
|
1534
1467
|
keywords = {
|
|
@@ -1542,7 +1475,7 @@ class PostJsonSearch(QueryStringSearch):
|
|
|
1542
1475
|
|
|
1543
1476
|
# provider product type specific conf
|
|
1544
1477
|
prep.product_type_def_params = self.get_product_type_def_params(
|
|
1545
|
-
product_type,
|
|
1478
|
+
product_type, format_variables=kwargs
|
|
1546
1479
|
)
|
|
1547
1480
|
|
|
1548
1481
|
# Add to the query, the queryable parameters set in the provider product type definition
|
|
@@ -1555,10 +1488,8 @@ class PostJsonSearch(QueryStringSearch):
|
|
|
1555
1488
|
and isinstance(self.config.metadata_mapping[k], list)
|
|
1556
1489
|
}
|
|
1557
1490
|
)
|
|
1558
|
-
if getattr(self.config, "dates_required", False):
|
|
1559
|
-
self._check_date_params(keywords, product_type)
|
|
1560
1491
|
|
|
1561
|
-
qp, _ = self.build_query_string(product_type,
|
|
1492
|
+
qp, _ = self.build_query_string(product_type, keywords)
|
|
1562
1493
|
|
|
1563
1494
|
for query_param, query_value in qp.items():
|
|
1564
1495
|
if (
|
|
@@ -1641,7 +1572,7 @@ class PostJsonSearch(QueryStringSearch):
|
|
|
1641
1572
|
|
|
1642
1573
|
def normalize_results(
|
|
1643
1574
|
self, results: RawSearchResult, **kwargs: Any
|
|
1644
|
-
) ->
|
|
1575
|
+
) -> list[EOProduct]:
|
|
1645
1576
|
"""Build EOProducts from provider results"""
|
|
1646
1577
|
normalized = super().normalize_results(results, **kwargs)
|
|
1647
1578
|
for product in normalized:
|
|
@@ -1676,12 +1607,12 @@ class PostJsonSearch(QueryStringSearch):
|
|
|
1676
1607
|
self,
|
|
1677
1608
|
prep: PreparedSearch = PreparedSearch(),
|
|
1678
1609
|
**kwargs: Any,
|
|
1679
|
-
) ->
|
|
1610
|
+
) -> tuple[list[str], Optional[int]]:
|
|
1680
1611
|
"""Adds pagination to query parameters, and auth to url"""
|
|
1681
1612
|
page = prep.page
|
|
1682
1613
|
items_per_page = prep.items_per_page
|
|
1683
1614
|
count = prep.count
|
|
1684
|
-
urls:
|
|
1615
|
+
urls: list[str] = []
|
|
1685
1616
|
total_results = 0 if count else None
|
|
1686
1617
|
|
|
1687
1618
|
if "count_endpoint" not in self.config.pagination:
|
|
@@ -1750,7 +1681,7 @@ class PostJsonSearch(QueryStringSearch):
|
|
|
1750
1681
|
raise ValidationError("Cannot request empty URL")
|
|
1751
1682
|
info_message = prep.info_message
|
|
1752
1683
|
exception_message = prep.exception_message
|
|
1753
|
-
timeout = getattr(self.config, "timeout",
|
|
1684
|
+
timeout = getattr(self.config, "timeout", DEFAULT_SEARCH_TIMEOUT)
|
|
1754
1685
|
ssl_verify = getattr(self.config, "ssl_verify", True)
|
|
1755
1686
|
try:
|
|
1756
1687
|
# auth if needed
|
|
@@ -1841,24 +1772,24 @@ class StacSearch(PostJsonSearch):
|
|
|
1841
1772
|
self.config.results_entry = results_entry
|
|
1842
1773
|
|
|
1843
1774
|
def build_query_string(
|
|
1844
|
-
self, product_type: str,
|
|
1845
|
-
) ->
|
|
1775
|
+
self, product_type: str, query_dict: dict[str, Any]
|
|
1776
|
+
) -> tuple[dict[str, Any], str]:
|
|
1846
1777
|
"""Build The query string using the search parameters"""
|
|
1847
1778
|
logger.debug("Building the query string that will be used for search")
|
|
1848
1779
|
|
|
1849
1780
|
# handle opened time intervals
|
|
1850
1781
|
if any(
|
|
1851
|
-
|
|
1852
|
-
for
|
|
1782
|
+
q in query_dict
|
|
1783
|
+
for q in ("startTimeFromAscendingNode", "completionTimeFromAscendingNode")
|
|
1853
1784
|
):
|
|
1854
|
-
|
|
1855
|
-
|
|
1785
|
+
query_dict.setdefault("startTimeFromAscendingNode", "..")
|
|
1786
|
+
query_dict.setdefault("completionTimeFromAscendingNode", "..")
|
|
1856
1787
|
|
|
1857
|
-
query_params = format_query_params(product_type, self.config,
|
|
1788
|
+
query_params = format_query_params(product_type, self.config, query_dict)
|
|
1858
1789
|
|
|
1859
1790
|
# Build the final query string, in one go without quoting it
|
|
1860
1791
|
# (some providers do not operate well with urlencoded and quoted query strings)
|
|
1861
|
-
def quote_via(x: Any, *_args, **_kwargs) -> str:
|
|
1792
|
+
def quote_via(x: Any, *_args: Any, **_kwargs: Any) -> str:
|
|
1862
1793
|
return x
|
|
1863
1794
|
|
|
1864
1795
|
return (
|
|
@@ -1868,7 +1799,7 @@ class StacSearch(PostJsonSearch):
|
|
|
1868
1799
|
|
|
1869
1800
|
def discover_queryables(
|
|
1870
1801
|
self, **kwargs: Any
|
|
1871
|
-
) -> Optional[
|
|
1802
|
+
) -> Optional[dict[str, Annotated[Any, FieldInfo]]]:
|
|
1872
1803
|
"""Fetch queryables list from provider using `discover_queryables` conf
|
|
1873
1804
|
|
|
1874
1805
|
:param kwargs: additional filters for queryables (`productType` and other search
|
|
@@ -1915,7 +1846,8 @@ class StacSearch(PostJsonSearch):
|
|
|
1915
1846
|
return None
|
|
1916
1847
|
|
|
1917
1848
|
fetch_url = unparsed_fetch_url.format(
|
|
1918
|
-
provider_product_type=provider_product_type,
|
|
1849
|
+
provider_product_type=provider_product_type,
|
|
1850
|
+
**self.config.__dict__,
|
|
1919
1851
|
)
|
|
1920
1852
|
auth = (
|
|
1921
1853
|
self.auth
|
|
@@ -1932,7 +1864,8 @@ class StacSearch(PostJsonSearch):
|
|
|
1932
1864
|
"{} {} instance:".format(self.provider, self.__class__.__name__),
|
|
1933
1865
|
),
|
|
1934
1866
|
)
|
|
1935
|
-
except (RequestError, KeyError, AttributeError):
|
|
1867
|
+
except (RequestError, KeyError, AttributeError) as e:
|
|
1868
|
+
logger.warning("failure in queryables discovery: %s", e)
|
|
1936
1869
|
return None
|
|
1937
1870
|
else:
|
|
1938
1871
|
json_queryables = dict()
|
|
@@ -1964,7 +1897,7 @@ class StacSearch(PostJsonSearch):
|
|
|
1964
1897
|
return None
|
|
1965
1898
|
|
|
1966
1899
|
# convert json results to pydantic model fields
|
|
1967
|
-
field_definitions:
|
|
1900
|
+
field_definitions: dict[str, Any] = dict()
|
|
1968
1901
|
for json_param, json_mtd in json_queryables.items():
|
|
1969
1902
|
param = (
|
|
1970
1903
|
get_queryable_from_provider(
|
|
@@ -1997,7 +1930,7 @@ class PostJsonSearchWithStacQueryables(StacSearch, PostJsonSearch):
|
|
|
1997
1930
|
PostJsonSearch.__init__(self, provider, config)
|
|
1998
1931
|
|
|
1999
1932
|
def build_query_string(
|
|
2000
|
-
self, product_type: str,
|
|
2001
|
-
) ->
|
|
1933
|
+
self, product_type: str, query_dict: dict[str, Any]
|
|
1934
|
+
) -> tuple[dict[str, Any], str]:
|
|
2002
1935
|
"""Build The query string using the search parameters"""
|
|
2003
|
-
return PostJsonSearch.build_query_string(self, product_type,
|
|
1936
|
+
return PostJsonSearch.build_query_string(self, product_type, query_dict)
|