eodag 3.0.1__py3-none-any.whl → 3.1.0b2__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- eodag/api/core.py +164 -127
- eodag/api/product/_assets.py +11 -11
- eodag/api/product/_product.py +45 -30
- eodag/api/product/drivers/__init__.py +81 -4
- eodag/api/product/drivers/base.py +65 -4
- eodag/api/product/drivers/generic.py +65 -0
- eodag/api/product/drivers/sentinel1.py +97 -0
- eodag/api/product/drivers/sentinel2.py +95 -0
- eodag/api/product/metadata_mapping.py +101 -85
- eodag/api/search_result.py +13 -23
- eodag/cli.py +26 -5
- eodag/config.py +78 -81
- eodag/plugins/apis/base.py +1 -1
- eodag/plugins/apis/ecmwf.py +46 -22
- eodag/plugins/apis/usgs.py +16 -15
- eodag/plugins/authentication/aws_auth.py +16 -13
- eodag/plugins/authentication/base.py +5 -3
- eodag/plugins/authentication/header.py +3 -3
- eodag/plugins/authentication/keycloak.py +4 -4
- eodag/plugins/authentication/oauth.py +7 -3
- eodag/plugins/authentication/openid_connect.py +16 -16
- eodag/plugins/authentication/sas_auth.py +4 -4
- eodag/plugins/authentication/token.py +41 -10
- eodag/plugins/authentication/token_exchange.py +1 -1
- eodag/plugins/base.py +4 -4
- eodag/plugins/crunch/base.py +4 -4
- eodag/plugins/crunch/filter_date.py +4 -4
- eodag/plugins/crunch/filter_latest_intersect.py +6 -6
- eodag/plugins/crunch/filter_latest_tpl_name.py +7 -7
- eodag/plugins/crunch/filter_overlap.py +4 -4
- eodag/plugins/crunch/filter_property.py +6 -7
- eodag/plugins/download/aws.py +58 -78
- eodag/plugins/download/base.py +38 -56
- eodag/plugins/download/creodias_s3.py +29 -0
- eodag/plugins/download/http.py +173 -183
- eodag/plugins/download/s3rest.py +10 -11
- eodag/plugins/manager.py +10 -20
- eodag/plugins/search/__init__.py +6 -5
- eodag/plugins/search/base.py +87 -44
- eodag/plugins/search/build_search_result.py +1067 -329
- eodag/plugins/search/cop_marine.py +22 -12
- eodag/plugins/search/creodias_s3.py +9 -73
- eodag/plugins/search/csw.py +11 -11
- eodag/plugins/search/data_request_search.py +16 -15
- eodag/plugins/search/qssearch.py +103 -187
- eodag/plugins/search/stac_list_assets.py +85 -0
- eodag/plugins/search/static_stac_search.py +3 -3
- eodag/resources/ext_product_types.json +1 -1
- eodag/resources/product_types.yml +663 -304
- eodag/resources/providers.yml +823 -1749
- eodag/resources/stac_api.yml +2 -2
- eodag/resources/user_conf_template.yml +11 -0
- eodag/rest/cache.py +2 -2
- eodag/rest/config.py +3 -3
- eodag/rest/core.py +112 -82
- eodag/rest/errors.py +5 -5
- eodag/rest/server.py +33 -14
- eodag/rest/stac.py +40 -38
- eodag/rest/types/collections_search.py +3 -3
- eodag/rest/types/eodag_search.py +29 -23
- eodag/rest/types/queryables.py +15 -16
- eodag/rest/types/stac_search.py +15 -25
- eodag/rest/utils/__init__.py +14 -21
- eodag/rest/utils/cql_evaluate.py +6 -6
- eodag/rest/utils/rfc3339.py +2 -2
- eodag/types/__init__.py +75 -28
- eodag/types/bbox.py +2 -2
- eodag/types/download_args.py +3 -3
- eodag/types/queryables.py +183 -72
- eodag/types/search_args.py +4 -4
- eodag/types/whoosh.py +127 -3
- eodag/utils/__init__.py +152 -50
- eodag/utils/exceptions.py +28 -21
- eodag/utils/import_system.py +2 -2
- eodag/utils/repr.py +65 -6
- eodag/utils/requests.py +13 -13
- eodag/utils/rest.py +2 -2
- eodag/utils/s3.py +208 -0
- eodag/utils/stac_reader.py +10 -10
- {eodag-3.0.1.dist-info → eodag-3.1.0b2.dist-info}/METADATA +77 -76
- eodag-3.1.0b2.dist-info/RECORD +113 -0
- {eodag-3.0.1.dist-info → eodag-3.1.0b2.dist-info}/WHEEL +1 -1
- {eodag-3.0.1.dist-info → eodag-3.1.0b2.dist-info}/entry_points.txt +4 -2
- eodag/utils/constraints.py +0 -244
- eodag-3.0.1.dist-info/RECORD +0 -109
- {eodag-3.0.1.dist-info → eodag-3.1.0b2.dist-info}/LICENSE +0 -0
- {eodag-3.0.1.dist-info → eodag-3.1.0b2.dist-info}/top_level.txt +0 -0
eodag/plugins/search/qssearch.py
CHANGED
|
@@ -20,18 +20,14 @@ from __future__ import annotations
|
|
|
20
20
|
import logging
|
|
21
21
|
import re
|
|
22
22
|
from copy import copy as copy_copy
|
|
23
|
-
from datetime import datetime
|
|
23
|
+
from datetime import datetime, timedelta
|
|
24
24
|
from typing import (
|
|
25
25
|
TYPE_CHECKING,
|
|
26
26
|
Annotated,
|
|
27
27
|
Any,
|
|
28
28
|
Callable,
|
|
29
|
-
Dict,
|
|
30
|
-
List,
|
|
31
29
|
Optional,
|
|
32
30
|
Sequence,
|
|
33
|
-
Set,
|
|
34
|
-
Tuple,
|
|
35
31
|
TypedDict,
|
|
36
32
|
cast,
|
|
37
33
|
get_args,
|
|
@@ -75,9 +71,10 @@ from eodag.api.search_result import RawSearchResult
|
|
|
75
71
|
from eodag.plugins.search import PreparedSearch
|
|
76
72
|
from eodag.plugins.search.base import Search
|
|
77
73
|
from eodag.types import json_field_definition_to_python, model_fields_to_annotated
|
|
78
|
-
from eodag.types.queryables import CommonQueryables
|
|
79
74
|
from eodag.types.search_args import SortByList
|
|
80
75
|
from eodag.utils import (
|
|
76
|
+
DEFAULT_MISSION_START_DATE,
|
|
77
|
+
DEFAULT_SEARCH_TIMEOUT,
|
|
81
78
|
GENERIC_PRODUCT_TYPE,
|
|
82
79
|
HTTP_REQ_TIMEOUT,
|
|
83
80
|
REQ_RETRY_BACKOFF_FACTOR,
|
|
@@ -94,10 +91,6 @@ from eodag.utils import (
|
|
|
94
91
|
update_nested_dict,
|
|
95
92
|
urlencode,
|
|
96
93
|
)
|
|
97
|
-
from eodag.utils.constraints import (
|
|
98
|
-
fetch_constraints,
|
|
99
|
-
get_constraint_queryables_with_additional_params,
|
|
100
|
-
)
|
|
101
94
|
from eodag.utils.exceptions import (
|
|
102
95
|
AuthenticationError,
|
|
103
96
|
MisconfiguredError,
|
|
@@ -132,7 +125,7 @@ class QueryStringSearch(Search):
|
|
|
132
125
|
authentication error; only used if ``need_auth=true``
|
|
133
126
|
* :attr:`~eodag.config.PluginConfig.ssl_verify` (``bool``): if the ssl certificates should be verified in
|
|
134
127
|
requests; default: ``True``
|
|
135
|
-
* :attr:`~eodag.config.PluginConfig.dont_quote` (``
|
|
128
|
+
* :attr:`~eodag.config.PluginConfig.dont_quote` (``list[str]``): characters that should not be quoted in the
|
|
136
129
|
url params
|
|
137
130
|
* :attr:`~eodag.config.PluginConfig.timeout` (``int``): time to wait until request timeout in seconds;
|
|
138
131
|
default: ``5``
|
|
@@ -140,10 +133,10 @@ class QueryStringSearch(Search):
|
|
|
140
133
|
total number of retries to allow; default: ``3``
|
|
141
134
|
* :attr:`~eodag.config.PluginConfig.retry_backoff_factor` (``int``): :class:`urllib3.util.Retry`
|
|
142
135
|
``backoff_factor`` parameter, backoff factor to apply between attempts after the second try; default: ``2``
|
|
143
|
-
* :attr:`~eodag.config.PluginConfig.retry_status_forcelist` (``
|
|
136
|
+
* :attr:`~eodag.config.PluginConfig.retry_status_forcelist` (``list[int]``): :class:`urllib3.util.Retry`
|
|
144
137
|
``status_forcelist`` parameter, list of integer HTTP status codes that we should force a retry on; default:
|
|
145
138
|
``[401, 429, 500, 502, 503, 504]``
|
|
146
|
-
* :attr:`~eodag.config.PluginConfig.literal_search_params` (``
|
|
139
|
+
* :attr:`~eodag.config.PluginConfig.literal_search_params` (``dict[str, str]``): A mapping of (search_param =>
|
|
147
140
|
search_value) pairs giving search parameters to be passed as is in the search url query string. This is useful
|
|
148
141
|
for example in situations where the user wants to add a fixed search query parameter exactly
|
|
149
142
|
as it is done on the provider interface.
|
|
@@ -187,10 +180,13 @@ class QueryStringSearch(Search):
|
|
|
187
180
|
* :attr:`~eodag.config.PluginConfig.DiscoverProductTypes.generic_product_type_id` (``str``): mapping for the
|
|
188
181
|
product type id
|
|
189
182
|
* :attr:`~eodag.config.PluginConfig.DiscoverProductTypes.generic_product_type_parsable_metadata`
|
|
190
|
-
(``
|
|
183
|
+
(``dict[str, str]``): mapping for product type metadata (e.g. ``abstract``, ``licence``) which can be parsed
|
|
191
184
|
from the provider result
|
|
192
185
|
* :attr:`~eodag.config.PluginConfig.DiscoverProductTypes.generic_product_type_parsable_properties`
|
|
193
|
-
(``
|
|
186
|
+
(``dict[str, str]``): mapping for product type properties which can be parsed from the result and are not
|
|
187
|
+
product type metadata
|
|
188
|
+
* :attr:`~eodag.config.PluginConfig.DiscoverProductTypes.generic_product_type_unparsable_properties`
|
|
189
|
+
(``dict[str, str]``): mapping for product type properties which cannot be parsed from the result and are not
|
|
194
190
|
product type metadata
|
|
195
191
|
* :attr:`~eodag.config.PluginConfig.DiscoverProductTypes.single_collection_fetch_url` (``str``): url to fetch
|
|
196
192
|
data for a single collection; used if product type metadata is not available from the endpoint given in
|
|
@@ -199,13 +195,13 @@ class QueryStringSearch(Search):
|
|
|
199
195
|
to be added to the :attr:`~eodag.config.PluginConfig.DiscoverProductTypes.fetch_url` to filter for a
|
|
200
196
|
collection
|
|
201
197
|
* :attr:`~eodag.config.PluginConfig.DiscoverProductTypes.single_product_type_parsable_metadata`
|
|
202
|
-
(``
|
|
198
|
+
(``dict[str, str]``): mapping for product type metadata returned by the endpoint given in
|
|
203
199
|
:attr:`~eodag.config.PluginConfig.DiscoverProductTypes.single_collection_fetch_url`.
|
|
204
200
|
|
|
205
201
|
* :attr:`~eodag.config.PluginConfig.sort` (:class:`~eodag.config.PluginConfig.Sort`): configuration for sorting
|
|
206
202
|
the results. It contains the keys:
|
|
207
203
|
|
|
208
|
-
* :attr:`~eodag.config.PluginConfig.Sort.sort_by_default` (``
|
|
204
|
+
* :attr:`~eodag.config.PluginConfig.Sort.sort_by_default` (``list[Tuple(str, Literal["ASC", "DESC"])]``):
|
|
209
205
|
parameter and sort order by which the result will be sorted by default (if the user does not enter a
|
|
210
206
|
``sort_by`` parameter); if not given the result will use the default sorting of the provider; Attention:
|
|
211
207
|
for some providers sorting might cause a timeout if no filters are used. In that case no default
|
|
@@ -221,12 +217,12 @@ class QueryStringSearch(Search):
|
|
|
221
217
|
* :attr:`~eodag.config.PluginConfig.Sort.sort_param_mapping` (``Dict [str, str]``): mapping for the parameters
|
|
222
218
|
available for sorting
|
|
223
219
|
* :attr:`~eodag.config.PluginConfig.Sort.sort_order_mapping`
|
|
224
|
-
(``
|
|
220
|
+
(``dict[Literal["ascending", "descending"], str]``): mapping for the sort order
|
|
225
221
|
* :attr:`~eodag.config.PluginConfig.Sort.max_sort_params` (``int``): maximum number of sort parameters
|
|
226
222
|
supported by the provider; used to validate the user input to avoid failed requests or unexpected behaviour
|
|
227
223
|
(not all parameters are used in the request)
|
|
228
224
|
|
|
229
|
-
* :attr:`~eodag.config.PluginConfig.metadata_mapping` (``
|
|
225
|
+
* :attr:`~eodag.config.PluginConfig.metadata_mapping` (``dict[str, Any]``): The search plugins of this kind can
|
|
230
226
|
detect when a metadata mapping is "query-able", and get the semantics of how to format the query string
|
|
231
227
|
parameter that enables to make a query on the corresponding metadata. To make a metadata query-able,
|
|
232
228
|
just configure it in the metadata mapping to be a list of 2 items, the first one being the
|
|
@@ -259,7 +255,7 @@ class QueryStringSearch(Search):
|
|
|
259
255
|
metadata is activated; default: ``False``; if false, the other parameters are not used;
|
|
260
256
|
* :attr:`~eodag.config.PluginConfig.DiscoverMetadata.metadata_pattern` (``str``): regex string a parameter in
|
|
261
257
|
the result should match so that is used
|
|
262
|
-
* :attr:`~eodag.config.PluginConfig.DiscoverMetadata.search_param` (``Union [str,
|
|
258
|
+
* :attr:`~eodag.config.PluginConfig.DiscoverMetadata.search_param` (``Union [str, dict[str, Any]]``): format
|
|
263
259
|
to add a query param given by the user and not in the metadata mapping to the requests, 'metadata' will be
|
|
264
260
|
replaced by the search param; can be a string or a dict containing
|
|
265
261
|
:attr:`~eodag.config.PluginConfig.free_text_search_operations`
|
|
@@ -282,16 +278,12 @@ class QueryStringSearch(Search):
|
|
|
282
278
|
|
|
283
279
|
* :attr:`~eodag.config.PluginConfig.constraints_file_url` (``str``): url to fetch the constraints for a specific
|
|
284
280
|
product type, can be an http url or a path to a file; the constraints are used to build queryables
|
|
285
|
-
* :attr:`~eodag.config.PluginConfig.constraints_file_dataset_key` (``str``): key which is used in the eodag
|
|
286
|
-
configuration to map the eodag product type to the provider product type; default: ``dataset``
|
|
287
281
|
* :attr:`~eodag.config.PluginConfig.constraints_entry` (``str``): key in the json result where the constraints
|
|
288
282
|
can be found; if not given, it is assumed that the constraints are on top level of the result, i.e.
|
|
289
283
|
the result is an array of constraints
|
|
290
|
-
* :attr:`~eodag.config.PluginConfig.stop_without_constraints_entry_key` (``bool``): if true only a provider
|
|
291
|
-
result containing `constraints_entry` is accepted as valid and used to create constraints; default: ``False``
|
|
292
284
|
"""
|
|
293
285
|
|
|
294
|
-
extract_properties:
|
|
286
|
+
extract_properties: dict[str, Callable[..., dict[str, Any]]] = {
|
|
295
287
|
"xml": properties_from_xml,
|
|
296
288
|
"json": properties_from_json,
|
|
297
289
|
}
|
|
@@ -302,8 +294,8 @@ class QueryStringSearch(Search):
|
|
|
302
294
|
self.config.__dict__.setdefault("results_entry", "features")
|
|
303
295
|
self.config.__dict__.setdefault("pagination", {})
|
|
304
296
|
self.config.__dict__.setdefault("free_text_search_operations", {})
|
|
305
|
-
self.search_urls:
|
|
306
|
-
self.query_params:
|
|
297
|
+
self.search_urls: list[str] = []
|
|
298
|
+
self.query_params: dict[str, str] = dict()
|
|
307
299
|
self.query_string = ""
|
|
308
300
|
self.next_page_url = None
|
|
309
301
|
self.next_page_query_obj = None
|
|
@@ -448,7 +440,7 @@ class QueryStringSearch(Search):
|
|
|
448
440
|
self.next_page_query_obj = None
|
|
449
441
|
self.next_page_merge = None
|
|
450
442
|
|
|
451
|
-
def discover_product_types(self, **kwargs: Any) -> Optional[
|
|
443
|
+
def discover_product_types(self, **kwargs: Any) -> Optional[dict[str, Any]]:
|
|
452
444
|
"""Fetch product types list from provider using `discover_product_types` conf
|
|
453
445
|
|
|
454
446
|
:returns: configuration dict containing fetched product types information
|
|
@@ -465,7 +457,7 @@ class QueryStringSearch(Search):
|
|
|
465
457
|
# no pagination
|
|
466
458
|
return self.discover_product_types_per_page(**kwargs)
|
|
467
459
|
|
|
468
|
-
conf_update_dict:
|
|
460
|
+
conf_update_dict: dict[str, Any] = {
|
|
469
461
|
"providers_config": {},
|
|
470
462
|
"product_types_config": {},
|
|
471
463
|
}
|
|
@@ -498,7 +490,7 @@ class QueryStringSearch(Search):
|
|
|
498
490
|
|
|
499
491
|
def discover_product_types_per_page(
|
|
500
492
|
self, **kwargs: Any
|
|
501
|
-
) -> Optional[
|
|
493
|
+
) -> Optional[dict[str, Any]]:
|
|
502
494
|
"""Fetch product types list from provider using `discover_product_types` conf
|
|
503
495
|
using paginated ``kwargs["fetch_url"]``
|
|
504
496
|
|
|
@@ -556,7 +548,7 @@ class QueryStringSearch(Search):
|
|
|
556
548
|
return None
|
|
557
549
|
else:
|
|
558
550
|
try:
|
|
559
|
-
conf_update_dict:
|
|
551
|
+
conf_update_dict: dict[str, Any] = {
|
|
560
552
|
"providers_config": {},
|
|
561
553
|
"product_types_config": {},
|
|
562
554
|
}
|
|
@@ -575,7 +567,7 @@ class QueryStringSearch(Search):
|
|
|
575
567
|
result = result[0]
|
|
576
568
|
|
|
577
569
|
def conf_update_from_product_type_result(
|
|
578
|
-
product_type_result:
|
|
570
|
+
product_type_result: dict[str, Any]
|
|
579
571
|
) -> None:
|
|
580
572
|
"""Update ``conf_update_dict`` using given product type json response"""
|
|
581
573
|
# providers_config extraction
|
|
@@ -641,7 +633,11 @@ class QueryStringSearch(Search):
|
|
|
641
633
|
][kf]
|
|
642
634
|
)
|
|
643
635
|
for kf in keywords_fields
|
|
644
|
-
if
|
|
636
|
+
if kf
|
|
637
|
+
in conf_update_dict["product_types_config"][
|
|
638
|
+
generic_product_type_id
|
|
639
|
+
]
|
|
640
|
+
and conf_update_dict["product_types_config"][
|
|
645
641
|
generic_product_type_id
|
|
646
642
|
][kf]
|
|
647
643
|
!= NOT_AVAILABLE
|
|
@@ -699,7 +695,7 @@ class QueryStringSearch(Search):
|
|
|
699
695
|
|
|
700
696
|
def _get_product_type_metadata_from_single_collection_endpoint(
|
|
701
697
|
self, product_type: str
|
|
702
|
-
) ->
|
|
698
|
+
) -> dict[str, Any]:
|
|
703
699
|
"""
|
|
704
700
|
retrieves additional product type information from an endpoint returning data for a single collection
|
|
705
701
|
:param product_type: product type
|
|
@@ -723,107 +719,11 @@ class QueryStringSearch(Search):
|
|
|
723
719
|
self.config.discover_product_types["single_product_type_parsable_metadata"],
|
|
724
720
|
)
|
|
725
721
|
|
|
726
|
-
def discover_queryables(
|
|
727
|
-
self, **kwargs: Any
|
|
728
|
-
) -> Optional[Dict[str, Annotated[Any, FieldInfo]]]:
|
|
729
|
-
"""Fetch queryables list from provider using its constraints file
|
|
730
|
-
|
|
731
|
-
:param kwargs: additional filters for queryables (`productType` and other search
|
|
732
|
-
arguments)
|
|
733
|
-
:returns: fetched queryable parameters dict
|
|
734
|
-
"""
|
|
735
|
-
product_type = kwargs.pop("productType", None)
|
|
736
|
-
if not product_type:
|
|
737
|
-
return {}
|
|
738
|
-
constraints_file_url = getattr(self.config, "constraints_file_url", "")
|
|
739
|
-
if not constraints_file_url:
|
|
740
|
-
return {}
|
|
741
|
-
|
|
742
|
-
constraints_file_dataset_key = getattr(
|
|
743
|
-
self.config, "constraints_file_dataset_key", "dataset"
|
|
744
|
-
)
|
|
745
|
-
provider_product_type = self.config.products.get(product_type, {}).get(
|
|
746
|
-
constraints_file_dataset_key, None
|
|
747
|
-
)
|
|
748
|
-
|
|
749
|
-
# defaults
|
|
750
|
-
default_queryables = self._get_defaults_as_queryables(product_type)
|
|
751
|
-
# remove unwanted queryables
|
|
752
|
-
for param in getattr(self.config, "remove_from_queryables", []):
|
|
753
|
-
default_queryables.pop(param, None)
|
|
754
|
-
|
|
755
|
-
non_empty_kwargs = {k: v for k, v in kwargs.items() if v}
|
|
756
|
-
|
|
757
|
-
if "{" in constraints_file_url:
|
|
758
|
-
constraints_file_url = constraints_file_url.format(
|
|
759
|
-
dataset=provider_product_type
|
|
760
|
-
)
|
|
761
|
-
constraints = fetch_constraints(constraints_file_url, self)
|
|
762
|
-
if not constraints:
|
|
763
|
-
return default_queryables
|
|
764
|
-
|
|
765
|
-
constraint_params: Dict[str, Dict[str, Set[Any]]] = {}
|
|
766
|
-
if len(kwargs) == 0:
|
|
767
|
-
# get values from constraints without additional filters
|
|
768
|
-
for constraint in constraints:
|
|
769
|
-
for key in constraint.keys():
|
|
770
|
-
if key in constraint_params:
|
|
771
|
-
constraint_params[key]["enum"].update(constraint[key])
|
|
772
|
-
else:
|
|
773
|
-
constraint_params[key] = {"enum": set(constraint[key])}
|
|
774
|
-
else:
|
|
775
|
-
# get values from constraints with additional filters
|
|
776
|
-
constraints_input_params = {k: v for k, v in non_empty_kwargs.items()}
|
|
777
|
-
constraint_params = get_constraint_queryables_with_additional_params(
|
|
778
|
-
constraints, constraints_input_params, self, product_type
|
|
779
|
-
)
|
|
780
|
-
# query params that are not in constraints but might be default queryables
|
|
781
|
-
if len(constraint_params) == 1 and "not_available" in constraint_params:
|
|
782
|
-
not_queryables = set()
|
|
783
|
-
for constraint_param in constraint_params["not_available"]["enum"]:
|
|
784
|
-
param = CommonQueryables.get_queryable_from_alias(constraint_param)
|
|
785
|
-
if param in dict(
|
|
786
|
-
CommonQueryables.model_fields, **default_queryables
|
|
787
|
-
):
|
|
788
|
-
non_empty_kwargs.pop(constraint_param)
|
|
789
|
-
else:
|
|
790
|
-
not_queryables.add(constraint_param)
|
|
791
|
-
if not_queryables:
|
|
792
|
-
raise ValidationError(
|
|
793
|
-
f"parameter(s) {str(not_queryables)} not queryable"
|
|
794
|
-
)
|
|
795
|
-
else:
|
|
796
|
-
# get constraints again without common queryables
|
|
797
|
-
constraint_params = (
|
|
798
|
-
get_constraint_queryables_with_additional_params(
|
|
799
|
-
constraints, non_empty_kwargs, self, product_type
|
|
800
|
-
)
|
|
801
|
-
)
|
|
802
|
-
|
|
803
|
-
field_definitions: Dict[str, Any] = dict()
|
|
804
|
-
for json_param, json_mtd in constraint_params.items():
|
|
805
|
-
param = (
|
|
806
|
-
get_queryable_from_provider(
|
|
807
|
-
json_param, self.get_metadata_mapping(product_type)
|
|
808
|
-
)
|
|
809
|
-
or json_param
|
|
810
|
-
)
|
|
811
|
-
default = kwargs.get(param, None) or self.config.products.get(
|
|
812
|
-
product_type, {}
|
|
813
|
-
).get(param, None)
|
|
814
|
-
annotated_def = json_field_definition_to_python(
|
|
815
|
-
json_mtd, default_value=default, required=True
|
|
816
|
-
)
|
|
817
|
-
field_definitions[param] = get_args(annotated_def)
|
|
818
|
-
|
|
819
|
-
python_queryables = create_model("m", **field_definitions).model_fields
|
|
820
|
-
return dict(default_queryables, **model_fields_to_annotated(python_queryables))
|
|
821
|
-
|
|
822
722
|
def query(
|
|
823
723
|
self,
|
|
824
724
|
prep: PreparedSearch = PreparedSearch(),
|
|
825
725
|
**kwargs: Any,
|
|
826
|
-
) ->
|
|
726
|
+
) -> tuple[list[EOProduct], Optional[int]]:
|
|
827
727
|
"""Perform a search on an OpenSearch-like interface
|
|
828
728
|
|
|
829
729
|
:param prep: Object collecting needed information for search.
|
|
@@ -903,14 +803,14 @@ class QueryStringSearch(Search):
|
|
|
903
803
|
reason="Simply run `self.config.metadata_mapping.update(metadata_mapping)` instead",
|
|
904
804
|
version="2.10.0",
|
|
905
805
|
)
|
|
906
|
-
def update_metadata_mapping(self, metadata_mapping:
|
|
806
|
+
def update_metadata_mapping(self, metadata_mapping: dict[str, Any]) -> None:
|
|
907
807
|
"""Update plugin metadata_mapping with input metadata_mapping configuration"""
|
|
908
808
|
if self.config.metadata_mapping:
|
|
909
809
|
self.config.metadata_mapping.update(metadata_mapping)
|
|
910
810
|
|
|
911
811
|
def build_query_string(
|
|
912
812
|
self, product_type: str, **kwargs: Any
|
|
913
|
-
) ->
|
|
813
|
+
) -> tuple[dict[str, Any], str]:
|
|
914
814
|
"""Build The query string using the search parameters"""
|
|
915
815
|
logger.debug("Building the query string that will be used for search")
|
|
916
816
|
query_params = format_query_params(product_type, self.config, kwargs)
|
|
@@ -929,7 +829,7 @@ class QueryStringSearch(Search):
|
|
|
929
829
|
self,
|
|
930
830
|
prep: PreparedSearch = PreparedSearch(page=None, items_per_page=None),
|
|
931
831
|
**kwargs: Any,
|
|
932
|
-
) ->
|
|
832
|
+
) -> tuple[list[str], Optional[int]]:
|
|
933
833
|
"""Build paginated urls"""
|
|
934
834
|
page = prep.page
|
|
935
835
|
items_per_page = prep.items_per_page
|
|
@@ -998,7 +898,7 @@ class QueryStringSearch(Search):
|
|
|
998
898
|
|
|
999
899
|
def do_search(
|
|
1000
900
|
self, prep: PreparedSearch = PreparedSearch(items_per_page=None), **kwargs: Any
|
|
1001
|
-
) ->
|
|
901
|
+
) -> list[Any]:
|
|
1002
902
|
"""Perform the actual search request.
|
|
1003
903
|
|
|
1004
904
|
If there is a specified number of items per page, return the results as soon
|
|
@@ -1015,7 +915,7 @@ class QueryStringSearch(Search):
|
|
|
1015
915
|
"total_items_nb_key_path"
|
|
1016
916
|
]
|
|
1017
917
|
|
|
1018
|
-
results:
|
|
918
|
+
results: list[Any] = []
|
|
1019
919
|
for search_url in prep.search_urls:
|
|
1020
920
|
single_search_prep = copy_copy(prep)
|
|
1021
921
|
single_search_prep.url = search_url
|
|
@@ -1138,9 +1038,13 @@ class QueryStringSearch(Search):
|
|
|
1138
1038
|
logger.debug(
|
|
1139
1039
|
"Could not extract total_items_nb from search results"
|
|
1140
1040
|
)
|
|
1141
|
-
if
|
|
1041
|
+
if (
|
|
1042
|
+
getattr(self.config, "merge_responses", False)
|
|
1043
|
+
and self.config.result_type == "json"
|
|
1044
|
+
):
|
|
1045
|
+
json_result = cast(list[dict[str, Any]], result)
|
|
1142
1046
|
results = (
|
|
1143
|
-
[dict(r, **
|
|
1047
|
+
[dict(r, **json_result[i]) for i, r in enumerate(results)]
|
|
1144
1048
|
if results
|
|
1145
1049
|
else result
|
|
1146
1050
|
)
|
|
@@ -1162,14 +1066,14 @@ class QueryStringSearch(Search):
|
|
|
1162
1066
|
|
|
1163
1067
|
def normalize_results(
|
|
1164
1068
|
self, results: RawSearchResult, **kwargs: Any
|
|
1165
|
-
) ->
|
|
1069
|
+
) -> list[EOProduct]:
|
|
1166
1070
|
"""Build EOProducts from provider results"""
|
|
1167
1071
|
normalize_remaining_count = len(results)
|
|
1168
1072
|
logger.debug(
|
|
1169
1073
|
"Adapting %s plugin results to eodag product representation"
|
|
1170
1074
|
% normalize_remaining_count
|
|
1171
1075
|
)
|
|
1172
|
-
products:
|
|
1076
|
+
products: list[EOProduct] = []
|
|
1173
1077
|
for result in results:
|
|
1174
1078
|
product = EOProduct(
|
|
1175
1079
|
self.provider,
|
|
@@ -1184,8 +1088,15 @@ class QueryStringSearch(Search):
|
|
|
1184
1088
|
product.properties = dict(
|
|
1185
1089
|
getattr(self.config, "product_type_config", {}), **product.properties
|
|
1186
1090
|
)
|
|
1187
|
-
# move assets from properties to product's attr
|
|
1188
|
-
product.
|
|
1091
|
+
# move assets from properties to product's attr, normalize keys & roles
|
|
1092
|
+
for key, asset in product.properties.pop("assets", {}).items():
|
|
1093
|
+
norm_key, asset["roles"] = product.driver.guess_asset_key_and_roles(
|
|
1094
|
+
asset.get("href", ""), product
|
|
1095
|
+
)
|
|
1096
|
+
if norm_key:
|
|
1097
|
+
product.assets[norm_key] = asset
|
|
1098
|
+
# sort assets
|
|
1099
|
+
product.assets.data = dict(sorted(product.assets.data.items()))
|
|
1189
1100
|
products.append(product)
|
|
1190
1101
|
return products
|
|
1191
1102
|
|
|
@@ -1227,7 +1138,7 @@ class QueryStringSearch(Search):
|
|
|
1227
1138
|
total_results = int(count_results)
|
|
1228
1139
|
return total_results
|
|
1229
1140
|
|
|
1230
|
-
def get_collections(self, prep: PreparedSearch, **kwargs: Any) ->
|
|
1141
|
+
def get_collections(self, prep: PreparedSearch, **kwargs: Any) -> tuple[str, ...]:
|
|
1231
1142
|
"""Get the collection to which the product belongs"""
|
|
1232
1143
|
# See https://earth.esa.int/web/sentinel/missions/sentinel-2/news/-
|
|
1233
1144
|
# /asset_publisher/Ac0d/content/change-of
|
|
@@ -1238,7 +1149,7 @@ class QueryStringSearch(Search):
|
|
|
1238
1149
|
not hasattr(prep, "product_type_def_params")
|
|
1239
1150
|
or not prep.product_type_def_params
|
|
1240
1151
|
):
|
|
1241
|
-
collections:
|
|
1152
|
+
collections: set[str] = set()
|
|
1242
1153
|
collection = getattr(self.config, "collection", None)
|
|
1243
1154
|
if collection is None:
|
|
1244
1155
|
try:
|
|
@@ -1280,7 +1191,7 @@ class QueryStringSearch(Search):
|
|
|
1280
1191
|
info_message = prep.info_message
|
|
1281
1192
|
exception_message = prep.exception_message
|
|
1282
1193
|
try:
|
|
1283
|
-
timeout = getattr(self.config, "timeout",
|
|
1194
|
+
timeout = getattr(self.config, "timeout", DEFAULT_SEARCH_TIMEOUT)
|
|
1284
1195
|
ssl_verify = getattr(self.config, "ssl_verify", True)
|
|
1285
1196
|
|
|
1286
1197
|
retry_total = getattr(self.config, "retry_total", REQ_RETRY_TOTAL)
|
|
@@ -1293,7 +1204,7 @@ class QueryStringSearch(Search):
|
|
|
1293
1204
|
|
|
1294
1205
|
ssl_ctx = get_ssl_context(ssl_verify)
|
|
1295
1206
|
# auth if needed
|
|
1296
|
-
kwargs:
|
|
1207
|
+
kwargs: dict[str, Any] = {}
|
|
1297
1208
|
if (
|
|
1298
1209
|
getattr(self.config, "need_auth", False)
|
|
1299
1210
|
and hasattr(prep, "auth")
|
|
@@ -1424,7 +1335,7 @@ class ODataV4Search(QueryStringSearch):
|
|
|
1424
1335
|
|
|
1425
1336
|
def do_search(
|
|
1426
1337
|
self, prep: PreparedSearch = PreparedSearch(), **kwargs: Any
|
|
1427
|
-
) ->
|
|
1338
|
+
) -> list[Any]:
|
|
1428
1339
|
"""A two step search can be performed if the metadata are not given into the search result"""
|
|
1429
1340
|
|
|
1430
1341
|
if getattr(self.config, "per_product_metadata_query", False):
|
|
@@ -1459,7 +1370,7 @@ class ODataV4Search(QueryStringSearch):
|
|
|
1459
1370
|
else:
|
|
1460
1371
|
return super(ODataV4Search, self).do_search(prep, **kwargs)
|
|
1461
1372
|
|
|
1462
|
-
def get_metadata_search_url(self, entity:
|
|
1373
|
+
def get_metadata_search_url(self, entity: dict[str, Any]) -> str:
|
|
1463
1374
|
"""Build the metadata link for the given entity"""
|
|
1464
1375
|
return "{}({})/Metadata".format(
|
|
1465
1376
|
self.config.api_endpoint.rstrip("/"), entity["id"]
|
|
@@ -1467,7 +1378,7 @@ class ODataV4Search(QueryStringSearch):
|
|
|
1467
1378
|
|
|
1468
1379
|
def normalize_results(
|
|
1469
1380
|
self, results: RawSearchResult, **kwargs: Any
|
|
1470
|
-
) ->
|
|
1381
|
+
) -> list[EOProduct]:
|
|
1471
1382
|
"""Build EOProducts from provider results
|
|
1472
1383
|
|
|
1473
1384
|
If configured, a metadata pre-mapping can be applied to simplify further metadata extraction.
|
|
@@ -1524,54 +1435,53 @@ class PostJsonSearch(QueryStringSearch):
|
|
|
1524
1435
|
"""
|
|
1525
1436
|
|
|
1526
1437
|
def _get_default_end_date_from_start_date(
|
|
1527
|
-
self, start_datetime: str,
|
|
1438
|
+
self, start_datetime: str, product_type_conf: dict[str, Any]
|
|
1528
1439
|
) -> str:
|
|
1529
|
-
default_end_date = self.config.products.get(product_type, {}).get(
|
|
1530
|
-
"_default_end_date", None
|
|
1531
|
-
)
|
|
1532
|
-
if default_end_date:
|
|
1533
|
-
return default_end_date
|
|
1534
1440
|
try:
|
|
1535
1441
|
start_date = datetime.fromisoformat(start_datetime)
|
|
1536
1442
|
except ValueError:
|
|
1537
1443
|
start_date = datetime.strptime(start_datetime, "%Y-%m-%dT%H:%M:%SZ")
|
|
1538
|
-
|
|
1539
|
-
|
|
1540
|
-
|
|
1541
|
-
|
|
1542
|
-
):
|
|
1543
|
-
mapping = product_type_conf["metadata_mapping"][
|
|
1544
|
-
"startTimeFromAscendingNode"
|
|
1545
|
-
]
|
|
1444
|
+
if "completionTimeFromAscendingNode" in product_type_conf:
|
|
1445
|
+
mapping = product_type_conf["completionTimeFromAscendingNode"]
|
|
1446
|
+
# if date is mapped to year/month/(day), use end_date = start_date else start_date + 1 day
|
|
1447
|
+
# (default dates are only needed for ecmwf products where selected timespans should not be too large)
|
|
1546
1448
|
if isinstance(mapping, list) and "year" in mapping[0]:
|
|
1547
|
-
# if date is mapped to year/month/(day), use end_date = start_date to avoid large requests
|
|
1548
1449
|
end_date = start_date
|
|
1549
|
-
|
|
1450
|
+
else:
|
|
1451
|
+
end_date = start_date + timedelta(days=1)
|
|
1452
|
+
return end_date.isoformat()
|
|
1550
1453
|
return self.get_product_type_cfg_value("missionEndDate", today().isoformat())
|
|
1551
1454
|
|
|
1552
|
-
def _check_date_params(
|
|
1455
|
+
def _check_date_params(
|
|
1456
|
+
self, keywords: dict[str, Any], product_type: Optional[str]
|
|
1457
|
+
) -> None:
|
|
1553
1458
|
"""checks if start and end date are present in the keywords and adds them if not"""
|
|
1554
1459
|
if (
|
|
1555
1460
|
"startTimeFromAscendingNode"
|
|
1556
1461
|
and "completionTimeFromAscendingNode" in keywords
|
|
1557
1462
|
):
|
|
1558
1463
|
return
|
|
1464
|
+
|
|
1465
|
+
product_type_conf = getattr(self.config, "metadata_mapping", {})
|
|
1466
|
+
if (
|
|
1467
|
+
product_type
|
|
1468
|
+
and product_type in self.config.products
|
|
1469
|
+
and "metadata_mapping" in self.config.products[product_type]
|
|
1470
|
+
):
|
|
1471
|
+
product_type_conf = self.config.products[product_type]["metadata_mapping"]
|
|
1559
1472
|
# start time given, end time missing
|
|
1560
1473
|
if "startTimeFromAscendingNode" in keywords:
|
|
1561
1474
|
keywords[
|
|
1562
1475
|
"completionTimeFromAscendingNode"
|
|
1563
1476
|
] = self._get_default_end_date_from_start_date(
|
|
1564
|
-
keywords["startTimeFromAscendingNode"],
|
|
1477
|
+
keywords["startTimeFromAscendingNode"], product_type_conf
|
|
1565
1478
|
)
|
|
1566
1479
|
return
|
|
1567
|
-
|
|
1568
|
-
if
|
|
1569
|
-
|
|
1570
|
-
|
|
1571
|
-
|
|
1572
|
-
mapping = product_type_conf["metadata_mapping"][
|
|
1573
|
-
"startTimeFromAscendingNode"
|
|
1574
|
-
]
|
|
1480
|
+
|
|
1481
|
+
if "completionTimeFromAscendingNode" in product_type_conf:
|
|
1482
|
+
mapping = product_type_conf["startTimeFromAscendingNode"]
|
|
1483
|
+
if not isinstance(mapping, list):
|
|
1484
|
+
mapping = product_type_conf["completionTimeFromAscendingNode"]
|
|
1575
1485
|
if isinstance(mapping, list):
|
|
1576
1486
|
# get time parameters (date, year, month, ...) from metadata mapping
|
|
1577
1487
|
input_mapping = mapping[0].replace("{{", "").replace("}}", "")
|
|
@@ -1587,25 +1497,26 @@ class PostJsonSearch(QueryStringSearch):
|
|
|
1587
1497
|
for tp in time_params:
|
|
1588
1498
|
if tp not in keywords:
|
|
1589
1499
|
in_keywords = False
|
|
1500
|
+
break
|
|
1590
1501
|
if not in_keywords:
|
|
1591
1502
|
keywords[
|
|
1592
1503
|
"startTimeFromAscendingNode"
|
|
1593
1504
|
] = self.get_product_type_cfg_value(
|
|
1594
|
-
"missionStartDate",
|
|
1505
|
+
"missionStartDate", DEFAULT_MISSION_START_DATE
|
|
1595
1506
|
)
|
|
1596
1507
|
keywords[
|
|
1597
1508
|
"completionTimeFromAscendingNode"
|
|
1598
1509
|
] = self._get_default_end_date_from_start_date(
|
|
1599
|
-
keywords["startTimeFromAscendingNode"],
|
|
1510
|
+
keywords["startTimeFromAscendingNode"], product_type_conf
|
|
1600
1511
|
)
|
|
1601
1512
|
|
|
1602
1513
|
def query(
|
|
1603
1514
|
self,
|
|
1604
1515
|
prep: PreparedSearch = PreparedSearch(),
|
|
1605
1516
|
**kwargs: Any,
|
|
1606
|
-
) ->
|
|
1517
|
+
) -> tuple[list[EOProduct], Optional[int]]:
|
|
1607
1518
|
"""Perform a search on an OpenSearch-like interface"""
|
|
1608
|
-
product_type = kwargs.get("productType",
|
|
1519
|
+
product_type = kwargs.get("productType", "")
|
|
1609
1520
|
count = prep.count
|
|
1610
1521
|
# remove "product_type" from search args if exists for compatibility with QueryStringSearch methods
|
|
1611
1522
|
kwargs.pop("product_type", None)
|
|
@@ -1720,6 +1631,7 @@ class PostJsonSearch(QueryStringSearch):
|
|
|
1720
1631
|
# do not try to extract total_items from search results if count is False
|
|
1721
1632
|
del prep.total_items_nb
|
|
1722
1633
|
del prep.need_count
|
|
1634
|
+
|
|
1723
1635
|
provider_results = self.do_search(prep, **kwargs)
|
|
1724
1636
|
if count and total_items is None and hasattr(prep, "total_items_nb"):
|
|
1725
1637
|
total_items = prep.total_items_nb
|
|
@@ -1733,7 +1645,7 @@ class PostJsonSearch(QueryStringSearch):
|
|
|
1733
1645
|
|
|
1734
1646
|
def normalize_results(
|
|
1735
1647
|
self, results: RawSearchResult, **kwargs: Any
|
|
1736
|
-
) ->
|
|
1648
|
+
) -> list[EOProduct]:
|
|
1737
1649
|
"""Build EOProducts from provider results"""
|
|
1738
1650
|
normalized = super().normalize_results(results, **kwargs)
|
|
1739
1651
|
for product in normalized:
|
|
@@ -1768,12 +1680,12 @@ class PostJsonSearch(QueryStringSearch):
|
|
|
1768
1680
|
self,
|
|
1769
1681
|
prep: PreparedSearch = PreparedSearch(),
|
|
1770
1682
|
**kwargs: Any,
|
|
1771
|
-
) ->
|
|
1683
|
+
) -> tuple[list[str], Optional[int]]:
|
|
1772
1684
|
"""Adds pagination to query parameters, and auth to url"""
|
|
1773
1685
|
page = prep.page
|
|
1774
1686
|
items_per_page = prep.items_per_page
|
|
1775
1687
|
count = prep.count
|
|
1776
|
-
urls:
|
|
1688
|
+
urls: list[str] = []
|
|
1777
1689
|
total_results = 0 if count else None
|
|
1778
1690
|
|
|
1779
1691
|
if "count_endpoint" not in self.config.pagination:
|
|
@@ -1842,7 +1754,7 @@ class PostJsonSearch(QueryStringSearch):
|
|
|
1842
1754
|
raise ValidationError("Cannot request empty URL")
|
|
1843
1755
|
info_message = prep.info_message
|
|
1844
1756
|
exception_message = prep.exception_message
|
|
1845
|
-
timeout = getattr(self.config, "timeout",
|
|
1757
|
+
timeout = getattr(self.config, "timeout", DEFAULT_SEARCH_TIMEOUT)
|
|
1846
1758
|
ssl_verify = getattr(self.config, "ssl_verify", True)
|
|
1847
1759
|
try:
|
|
1848
1760
|
# auth if needed
|
|
@@ -1934,7 +1846,7 @@ class StacSearch(PostJsonSearch):
|
|
|
1934
1846
|
|
|
1935
1847
|
def build_query_string(
|
|
1936
1848
|
self, product_type: str, **kwargs: Any
|
|
1937
|
-
) ->
|
|
1849
|
+
) -> tuple[dict[str, Any], str]:
|
|
1938
1850
|
"""Build The query string using the search parameters"""
|
|
1939
1851
|
logger.debug("Building the query string that will be used for search")
|
|
1940
1852
|
|
|
@@ -1960,7 +1872,7 @@ class StacSearch(PostJsonSearch):
|
|
|
1960
1872
|
|
|
1961
1873
|
def discover_queryables(
|
|
1962
1874
|
self, **kwargs: Any
|
|
1963
|
-
) -> Optional[
|
|
1875
|
+
) -> Optional[dict[str, Annotated[Any, FieldInfo]]]:
|
|
1964
1876
|
"""Fetch queryables list from provider using `discover_queryables` conf
|
|
1965
1877
|
|
|
1966
1878
|
:param kwargs: additional filters for queryables (`productType` and other search
|
|
@@ -2056,7 +1968,7 @@ class StacSearch(PostJsonSearch):
|
|
|
2056
1968
|
return None
|
|
2057
1969
|
|
|
2058
1970
|
# convert json results to pydantic model fields
|
|
2059
|
-
field_definitions:
|
|
1971
|
+
field_definitions: dict[str, Any] = dict()
|
|
2060
1972
|
for json_param, json_mtd in json_queryables.items():
|
|
2061
1973
|
param = (
|
|
2062
1974
|
get_queryable_from_provider(
|
|
@@ -2072,6 +1984,10 @@ class StacSearch(PostJsonSearch):
|
|
|
2072
1984
|
field_definitions[param] = get_args(annotated_def)
|
|
2073
1985
|
|
|
2074
1986
|
python_queryables = create_model("m", **field_definitions).model_fields
|
|
1987
|
+
# replace geometry by geom
|
|
1988
|
+
geom_queryable = python_queryables.pop("geometry", None)
|
|
1989
|
+
if geom_queryable:
|
|
1990
|
+
python_queryables["geom"] = geom_queryable
|
|
2075
1991
|
|
|
2076
1992
|
return model_fields_to_annotated(python_queryables)
|
|
2077
1993
|
|
|
@@ -2086,6 +2002,6 @@ class PostJsonSearchWithStacQueryables(StacSearch, PostJsonSearch):
|
|
|
2086
2002
|
|
|
2087
2003
|
def build_query_string(
|
|
2088
2004
|
self, product_type: str, **kwargs: Any
|
|
2089
|
-
) ->
|
|
2005
|
+
) -> tuple[dict[str, Any], str]:
|
|
2090
2006
|
"""Build The query string using the search parameters"""
|
|
2091
2007
|
return PostJsonSearch.build_query_string(self, product_type, **kwargs)
|