eodag 3.1.0b1__py3-none-any.whl → 3.1.0b2__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- eodag/api/core.py +59 -52
- eodag/api/product/_assets.py +5 -5
- eodag/api/product/_product.py +27 -12
- eodag/api/product/drivers/__init__.py +81 -4
- eodag/api/product/drivers/base.py +65 -4
- eodag/api/product/drivers/generic.py +65 -0
- eodag/api/product/drivers/sentinel1.py +97 -0
- eodag/api/product/drivers/sentinel2.py +95 -0
- eodag/api/product/metadata_mapping.py +62 -74
- eodag/api/search_result.py +13 -23
- eodag/cli.py +4 -4
- eodag/config.py +66 -69
- eodag/plugins/apis/base.py +1 -1
- eodag/plugins/apis/ecmwf.py +10 -9
- eodag/plugins/apis/usgs.py +11 -10
- eodag/plugins/authentication/aws_auth.py +16 -13
- eodag/plugins/authentication/base.py +5 -3
- eodag/plugins/authentication/header.py +3 -3
- eodag/plugins/authentication/keycloak.py +4 -4
- eodag/plugins/authentication/oauth.py +7 -3
- eodag/plugins/authentication/openid_connect.py +14 -14
- eodag/plugins/authentication/sas_auth.py +4 -4
- eodag/plugins/authentication/token.py +7 -7
- eodag/plugins/authentication/token_exchange.py +1 -1
- eodag/plugins/base.py +4 -4
- eodag/plugins/crunch/base.py +4 -4
- eodag/plugins/crunch/filter_date.py +4 -4
- eodag/plugins/crunch/filter_latest_intersect.py +6 -6
- eodag/plugins/crunch/filter_latest_tpl_name.py +7 -7
- eodag/plugins/crunch/filter_overlap.py +4 -4
- eodag/plugins/crunch/filter_property.py +4 -4
- eodag/plugins/download/aws.py +47 -66
- eodag/plugins/download/base.py +8 -17
- eodag/plugins/download/creodias_s3.py +2 -2
- eodag/plugins/download/http.py +30 -32
- eodag/plugins/download/s3rest.py +5 -4
- eodag/plugins/manager.py +10 -20
- eodag/plugins/search/__init__.py +6 -5
- eodag/plugins/search/base.py +35 -40
- eodag/plugins/search/build_search_result.py +69 -68
- eodag/plugins/search/cop_marine.py +22 -12
- eodag/plugins/search/creodias_s3.py +8 -78
- eodag/plugins/search/csw.py +11 -11
- eodag/plugins/search/data_request_search.py +16 -15
- eodag/plugins/search/qssearch.py +56 -52
- eodag/plugins/search/stac_list_assets.py +85 -0
- eodag/plugins/search/static_stac_search.py +3 -3
- eodag/resources/ext_product_types.json +1 -1
- eodag/resources/product_types.yml +288 -288
- eodag/resources/providers.yml +146 -6
- eodag/resources/stac_api.yml +2 -2
- eodag/resources/user_conf_template.yml +11 -0
- eodag/rest/cache.py +2 -2
- eodag/rest/config.py +3 -3
- eodag/rest/core.py +24 -24
- eodag/rest/errors.py +5 -5
- eodag/rest/server.py +3 -11
- eodag/rest/stac.py +40 -38
- eodag/rest/types/collections_search.py +3 -3
- eodag/rest/types/eodag_search.py +23 -23
- eodag/rest/types/queryables.py +13 -13
- eodag/rest/types/stac_search.py +15 -25
- eodag/rest/utils/__init__.py +11 -21
- eodag/rest/utils/cql_evaluate.py +6 -6
- eodag/rest/utils/rfc3339.py +2 -2
- eodag/types/__init__.py +24 -18
- eodag/types/bbox.py +2 -2
- eodag/types/download_args.py +2 -2
- eodag/types/queryables.py +5 -2
- eodag/types/search_args.py +4 -4
- eodag/types/whoosh.py +1 -3
- eodag/utils/__init__.py +81 -40
- eodag/utils/exceptions.py +2 -2
- eodag/utils/import_system.py +2 -2
- eodag/utils/requests.py +2 -2
- eodag/utils/rest.py +2 -2
- eodag/utils/s3.py +208 -0
- eodag/utils/stac_reader.py +10 -10
- {eodag-3.1.0b1.dist-info → eodag-3.1.0b2.dist-info}/METADATA +5 -4
- eodag-3.1.0b2.dist-info/RECORD +113 -0
- {eodag-3.1.0b1.dist-info → eodag-3.1.0b2.dist-info}/entry_points.txt +1 -0
- eodag-3.1.0b1.dist-info/RECORD +0 -108
- {eodag-3.1.0b1.dist-info → eodag-3.1.0b2.dist-info}/LICENSE +0 -0
- {eodag-3.1.0b1.dist-info → eodag-3.1.0b2.dist-info}/WHEEL +0 -0
- {eodag-3.1.0b1.dist-info → eodag-3.1.0b2.dist-info}/top_level.txt +0 -0
|
@@ -23,18 +23,7 @@ import logging
|
|
|
23
23
|
import re
|
|
24
24
|
from collections import OrderedDict
|
|
25
25
|
from datetime import datetime, timedelta
|
|
26
|
-
from typing import
|
|
27
|
-
TYPE_CHECKING,
|
|
28
|
-
Annotated,
|
|
29
|
-
Any,
|
|
30
|
-
Dict,
|
|
31
|
-
List,
|
|
32
|
-
Optional,
|
|
33
|
-
Set,
|
|
34
|
-
Tuple,
|
|
35
|
-
Union,
|
|
36
|
-
cast,
|
|
37
|
-
)
|
|
26
|
+
from typing import TYPE_CHECKING, Annotated, Any, Optional, Union, cast
|
|
38
27
|
from urllib.parse import quote_plus, unquote_plus
|
|
39
28
|
|
|
40
29
|
import geojson
|
|
@@ -61,9 +50,9 @@ from eodag.api.search_result import RawSearchResult
|
|
|
61
50
|
from eodag.plugins.search import PreparedSearch
|
|
62
51
|
from eodag.plugins.search.qssearch import PostJsonSearch, QueryStringSearch
|
|
63
52
|
from eodag.types import json_field_definition_to_python
|
|
64
|
-
from eodag.types.queryables import Queryables
|
|
53
|
+
from eodag.types.queryables import Queryables, QueryablesDict
|
|
65
54
|
from eodag.utils import (
|
|
66
|
-
|
|
55
|
+
DEFAULT_SEARCH_TIMEOUT,
|
|
67
56
|
deepcopy,
|
|
68
57
|
dict_items_recursive_sort,
|
|
69
58
|
get_geometry_from_various,
|
|
@@ -205,8 +194,8 @@ COP_DS_KEYWORDS = [
|
|
|
205
194
|
|
|
206
195
|
|
|
207
196
|
def keywords_to_mdt(
|
|
208
|
-
keywords:
|
|
209
|
-
) ->
|
|
197
|
+
keywords: list[str], prefix: Optional[str] = None
|
|
198
|
+
) -> dict[str, Any]:
|
|
210
199
|
"""
|
|
211
200
|
Make metadata mapping dict from a list of keywords
|
|
212
201
|
|
|
@@ -223,7 +212,7 @@ def keywords_to_mdt(
|
|
|
223
212
|
:param prefix: prefix to be added to the parameter in the mapping
|
|
224
213
|
:return: metadata mapping dict
|
|
225
214
|
"""
|
|
226
|
-
mdt:
|
|
215
|
+
mdt: dict[str, Any] = {}
|
|
227
216
|
for keyword in keywords:
|
|
228
217
|
key = f"{prefix}:{keyword}" if prefix else keyword
|
|
229
218
|
mdt[key] = [keyword, f'$."{key}"']
|
|
@@ -237,6 +226,8 @@ def strip_quotes(value: Any) -> Any:
|
|
|
237
226
|
'abc'
|
|
238
227
|
>>> strip_quotes(["'abc'", '"def'])
|
|
239
228
|
['abc', 'def']
|
|
229
|
+
>>> strip_quotes({"'abc'": 'def"'})
|
|
230
|
+
{'abc': 'def'}
|
|
240
231
|
|
|
241
232
|
:param value: value from which quotes should be removed (should be either str or list)
|
|
242
233
|
:return: value without quotes
|
|
@@ -245,13 +236,13 @@ def strip_quotes(value: Any) -> Any:
|
|
|
245
236
|
if isinstance(value, (list, tuple)):
|
|
246
237
|
return [strip_quotes(v) for v in value]
|
|
247
238
|
elif isinstance(value, dict):
|
|
248
|
-
|
|
239
|
+
return {strip_quotes(k): strip_quotes(v) for k, v in value.items()}
|
|
249
240
|
else:
|
|
250
241
|
return str(value).strip("'\"")
|
|
251
242
|
|
|
252
243
|
|
|
253
244
|
def _update_properties_from_element(
|
|
254
|
-
prop:
|
|
245
|
+
prop: dict[str, Any], element: dict[str, Any], values: list[str]
|
|
255
246
|
) -> None:
|
|
256
247
|
"""updates a property dict with the given values based on the information from the element dict
|
|
257
248
|
e.g. the type is set based on the type of the element
|
|
@@ -333,7 +324,7 @@ class ECMWFSearch(PostJsonSearch):
|
|
|
333
324
|
:param provider: An eodag providers configuration dictionary
|
|
334
325
|
:param config: Search plugin configuration:
|
|
335
326
|
|
|
336
|
-
* :attr:`~eodag.config.PluginConfig.remove_from_query` (``
|
|
327
|
+
* :attr:`~eodag.config.PluginConfig.remove_from_query` (``list[str]``): List of parameters
|
|
337
328
|
used to parse metadata but that must not be included to the query
|
|
338
329
|
* :attr:`~eodag.config.PluginConfig.end_date_excluded` (``bool``): Set to `False` if
|
|
339
330
|
provider does not include end date to search
|
|
@@ -350,9 +341,6 @@ class ECMWFSearch(PostJsonSearch):
|
|
|
350
341
|
"""
|
|
351
342
|
|
|
352
343
|
def __init__(self, provider: str, config: PluginConfig) -> None:
|
|
353
|
-
# cache fetching method
|
|
354
|
-
self.fetch_data = functools.lru_cache()(self._fetch_data)
|
|
355
|
-
|
|
356
344
|
config.metadata_mapping = {
|
|
357
345
|
**keywords_to_mdt(ECMWF_KEYWORDS + COP_DS_KEYWORDS, "ecmwf"),
|
|
358
346
|
**config.metadata_mapping,
|
|
@@ -402,7 +390,7 @@ class ECMWFSearch(PostJsonSearch):
|
|
|
402
390
|
"metadata_mapping"
|
|
403
391
|
] = product_type_metadata_mapping
|
|
404
392
|
|
|
405
|
-
def do_search(self, *args: Any, **kwargs: Any) ->
|
|
393
|
+
def do_search(self, *args: Any, **kwargs: Any) -> list[dict[str, Any]]:
|
|
406
394
|
"""Should perform the actual search request.
|
|
407
395
|
|
|
408
396
|
:param args: arguments to be used in the search
|
|
@@ -416,7 +404,7 @@ class ECMWFSearch(PostJsonSearch):
|
|
|
416
404
|
self,
|
|
417
405
|
prep: PreparedSearch = PreparedSearch(),
|
|
418
406
|
**kwargs: Any,
|
|
419
|
-
) ->
|
|
407
|
+
) -> tuple[list[EOProduct], Optional[int]]:
|
|
420
408
|
"""Build ready-to-download SearchResult
|
|
421
409
|
|
|
422
410
|
:param prep: :class:`~eodag.plugins.search.PreparedSearch` object containing information needed for the search
|
|
@@ -439,7 +427,7 @@ class ECMWFSearch(PostJsonSearch):
|
|
|
439
427
|
|
|
440
428
|
def build_query_string(
|
|
441
429
|
self, product_type: str, **kwargs: Any
|
|
442
|
-
) ->
|
|
430
|
+
) -> tuple[dict[str, Any], str]:
|
|
443
431
|
"""Build The query string using the search parameters
|
|
444
432
|
|
|
445
433
|
:param product_type: product type id
|
|
@@ -464,7 +452,7 @@ class ECMWFSearch(PostJsonSearch):
|
|
|
464
452
|
)
|
|
465
453
|
|
|
466
454
|
def _preprocess_search_params(
|
|
467
|
-
self, params:
|
|
455
|
+
self, params: dict[str, Any], product_type: Optional[str]
|
|
468
456
|
) -> None:
|
|
469
457
|
"""Preprocess search parameters before making a request to the CDS API.
|
|
470
458
|
|
|
@@ -543,9 +531,23 @@ class ECMWFSearch(PostJsonSearch):
|
|
|
543
531
|
if "geometry" in params:
|
|
544
532
|
params["geometry"] = get_geometry_from_various(geometry=params["geometry"])
|
|
545
533
|
|
|
534
|
+
def _get_product_type_queryables(
|
|
535
|
+
self, product_type: Optional[str], alias: Optional[str], filters: dict[str, Any]
|
|
536
|
+
) -> QueryablesDict:
|
|
537
|
+
"""Override to set additional_properties to false."""
|
|
538
|
+
default_values: dict[str, Any] = deepcopy(
|
|
539
|
+
getattr(self.config, "products", {}).get(product_type, {})
|
|
540
|
+
)
|
|
541
|
+
default_values.pop("metadata_mapping", None)
|
|
542
|
+
|
|
543
|
+
filters["productType"] = product_type
|
|
544
|
+
queryables = self.discover_queryables(**{**default_values, **filters}) or {}
|
|
545
|
+
|
|
546
|
+
return QueryablesDict(additional_properties=False, **queryables)
|
|
547
|
+
|
|
546
548
|
def discover_queryables(
|
|
547
549
|
self, **kwargs: Any
|
|
548
|
-
) -> Optional[
|
|
550
|
+
) -> Optional[dict[str, Annotated[Any, FieldInfo]]]:
|
|
549
551
|
"""Fetch queryables list from provider using its constraints file
|
|
550
552
|
|
|
551
553
|
:param kwargs: additional filters for queryables (`productType` and other search
|
|
@@ -571,13 +573,13 @@ class ECMWFSearch(PostJsonSearch):
|
|
|
571
573
|
getattr(self.config, "discover_queryables", {}).get("constraints_url", ""),
|
|
572
574
|
**kwargs,
|
|
573
575
|
)
|
|
574
|
-
constraints:
|
|
576
|
+
constraints: list[dict[str, Any]] = self._fetch_data(constraints_url)
|
|
575
577
|
|
|
576
578
|
form_url = format_metadata(
|
|
577
579
|
getattr(self.config, "discover_queryables", {}).get("form_url", ""),
|
|
578
580
|
**kwargs,
|
|
579
581
|
)
|
|
580
|
-
form = self.
|
|
582
|
+
form: list[dict[str, Any]] = self._fetch_data(form_url)
|
|
581
583
|
|
|
582
584
|
formated_kwargs = self.format_as_provider_keyword(
|
|
583
585
|
product_type, processed_kwargs
|
|
@@ -600,18 +602,18 @@ class ECMWFSearch(PostJsonSearch):
|
|
|
600
602
|
# we use non empty kwargs as default to integrate user inputs
|
|
601
603
|
# it is needed because pydantic json schema does not represent "value"
|
|
602
604
|
# but only "default"
|
|
603
|
-
non_empty_formated:
|
|
605
|
+
non_empty_formated: dict[str, Any] = {
|
|
604
606
|
k: v
|
|
605
607
|
for k, v in formated_kwargs.items()
|
|
606
608
|
if v and (not isinstance(v, list) or all(v))
|
|
607
609
|
}
|
|
608
|
-
non_empty_kwargs:
|
|
610
|
+
non_empty_kwargs: dict[str, Any] = {
|
|
609
611
|
k: v
|
|
610
612
|
for k, v in processed_kwargs.items()
|
|
611
613
|
if v and (not isinstance(v, list) or all(v))
|
|
612
614
|
}
|
|
613
615
|
|
|
614
|
-
required_keywords:
|
|
616
|
+
required_keywords: set[str] = set()
|
|
615
617
|
|
|
616
618
|
# calculate available values
|
|
617
619
|
if constraints:
|
|
@@ -634,7 +636,7 @@ class ECMWFSearch(PostJsonSearch):
|
|
|
634
636
|
return self.queryables_from_metadata_mapping(product_type)
|
|
635
637
|
if "{" in values_url:
|
|
636
638
|
values_url = values_url.format(productType=provider_product_type)
|
|
637
|
-
data = self.
|
|
639
|
+
data = self._fetch_data(values_url)
|
|
638
640
|
available_values = data["constraints"]
|
|
639
641
|
required_keywords = data.get("required", [])
|
|
640
642
|
|
|
@@ -651,7 +653,9 @@ class ECMWFSearch(PostJsonSearch):
|
|
|
651
653
|
"completionTimeFromAscendingNode",
|
|
652
654
|
"geom",
|
|
653
655
|
}
|
|
654
|
-
and keyword
|
|
656
|
+
and keyword not in [f["name"] for f in form]
|
|
657
|
+
and keyword.replace("ecmwf:", "")
|
|
658
|
+
not in set(list(available_values.keys()) + [f["name"] for f in form])
|
|
655
659
|
):
|
|
656
660
|
raise ValidationError(f"{keyword} is not a queryable parameter")
|
|
657
661
|
|
|
@@ -689,7 +693,7 @@ class ECMWFSearch(PostJsonSearch):
|
|
|
689
693
|
# area is geom in EODAG.
|
|
690
694
|
if queryables.pop("area", None):
|
|
691
695
|
queryables["geom"] = Annotated[
|
|
692
|
-
Union[str,
|
|
696
|
+
Union[str, dict[str, float], BaseGeometry],
|
|
693
697
|
Field(
|
|
694
698
|
None,
|
|
695
699
|
description="Read EODAG documentation for all supported geometry format.",
|
|
@@ -700,10 +704,10 @@ class ECMWFSearch(PostJsonSearch):
|
|
|
700
704
|
|
|
701
705
|
def available_values_from_constraints(
|
|
702
706
|
self,
|
|
703
|
-
constraints: list[
|
|
704
|
-
input_keywords:
|
|
705
|
-
form_keywords:
|
|
706
|
-
) ->
|
|
707
|
+
constraints: list[dict[str, Any]],
|
|
708
|
+
input_keywords: dict[str, Any],
|
|
709
|
+
form_keywords: list[str],
|
|
710
|
+
) -> dict[str, list[str]]:
|
|
707
711
|
"""
|
|
708
712
|
Filter constraints using input_keywords. Return list of available queryables.
|
|
709
713
|
All constraint entries must have the same parameters.
|
|
@@ -727,9 +731,9 @@ class ECMWFSearch(PostJsonSearch):
|
|
|
727
731
|
)
|
|
728
732
|
|
|
729
733
|
# filter constraint entries matching input keyword values
|
|
730
|
-
filtered_constraints:
|
|
734
|
+
filtered_constraints: list[dict[str, Any]]
|
|
731
735
|
|
|
732
|
-
parsed_keywords:
|
|
736
|
+
parsed_keywords: list[str] = []
|
|
733
737
|
for keyword in ordered_keywords:
|
|
734
738
|
values = input_keywords.get(keyword)
|
|
735
739
|
|
|
@@ -808,7 +812,7 @@ class ECMWFSearch(PostJsonSearch):
|
|
|
808
812
|
parsed_keywords.append(keyword)
|
|
809
813
|
constraints = filtered_constraints
|
|
810
814
|
|
|
811
|
-
available_values:
|
|
815
|
+
available_values: dict[str, Any] = {k: set() for k in ordered_keywords}
|
|
812
816
|
|
|
813
817
|
# we aggregate the constraint entries left
|
|
814
818
|
for entry in constraints:
|
|
@@ -819,10 +823,10 @@ class ECMWFSearch(PostJsonSearch):
|
|
|
819
823
|
|
|
820
824
|
def queryables_by_form(
|
|
821
825
|
self,
|
|
822
|
-
form:
|
|
823
|
-
available_values:
|
|
824
|
-
defaults:
|
|
825
|
-
) ->
|
|
826
|
+
form: list[dict[str, Any]],
|
|
827
|
+
available_values: dict[str, list[str]],
|
|
828
|
+
defaults: dict[str, Any],
|
|
829
|
+
) -> dict[str, Annotated[Any, FieldInfo]]:
|
|
826
830
|
"""
|
|
827
831
|
Generate Annotated field definitions from form entries and available values
|
|
828
832
|
Used by Copernicus services like cop_cds, cop_ads, cop_ewds.
|
|
@@ -832,9 +836,9 @@ class ECMWFSearch(PostJsonSearch):
|
|
|
832
836
|
:param defaults: default values for the parameters
|
|
833
837
|
:return: dict of annotated queryables
|
|
834
838
|
"""
|
|
835
|
-
queryables:
|
|
839
|
+
queryables: dict[str, Annotated[Any, FieldInfo]] = {}
|
|
836
840
|
|
|
837
|
-
required_list:
|
|
841
|
+
required_list: list[str] = []
|
|
838
842
|
for element in form:
|
|
839
843
|
name: str = element["name"]
|
|
840
844
|
|
|
@@ -869,9 +873,6 @@ class ECMWFSearch(PostJsonSearch):
|
|
|
869
873
|
if fields and (comment := fields[0].get("comment")):
|
|
870
874
|
prop["description"] = comment
|
|
871
875
|
|
|
872
|
-
if d := details.get("default"):
|
|
873
|
-
default = default or (d[0] if fields else d)
|
|
874
|
-
|
|
875
876
|
if name == "area" and isinstance(default, dict):
|
|
876
877
|
default = list(default.values())
|
|
877
878
|
|
|
@@ -901,10 +902,10 @@ class ECMWFSearch(PostJsonSearch):
|
|
|
901
902
|
|
|
902
903
|
def queryables_by_values(
|
|
903
904
|
self,
|
|
904
|
-
available_values:
|
|
905
|
-
required_keywords:
|
|
906
|
-
defaults:
|
|
907
|
-
) ->
|
|
905
|
+
available_values: dict[str, list[str]],
|
|
906
|
+
required_keywords: list[str],
|
|
907
|
+
defaults: dict[str, Any],
|
|
908
|
+
) -> dict[str, Annotated[Any, FieldInfo]]:
|
|
908
909
|
"""
|
|
909
910
|
Generate Annotated field definitions from available values.
|
|
910
911
|
Used by ECMWF data providers like dedt_lumi.
|
|
@@ -918,7 +919,7 @@ class ECMWFSearch(PostJsonSearch):
|
|
|
918
919
|
# Needed to map constraints like "xxxx" to eodag parameter "ecmwf:xxxx"
|
|
919
920
|
required = [ecmwf_format(k) for k in required_keywords]
|
|
920
921
|
|
|
921
|
-
queryables:
|
|
922
|
+
queryables: dict[str, Annotated[Any, FieldInfo]] = {}
|
|
922
923
|
for name, values in available_values.items():
|
|
923
924
|
# Rename keywords from form with metadata mapping.
|
|
924
925
|
# Needed to map constraints like "xxxx" to eodag parameter "ecmwf:xxxx"
|
|
@@ -939,8 +940,8 @@ class ECMWFSearch(PostJsonSearch):
|
|
|
939
940
|
return queryables
|
|
940
941
|
|
|
941
942
|
def format_as_provider_keyword(
|
|
942
|
-
self, product_type: str, properties:
|
|
943
|
-
) ->
|
|
943
|
+
self, product_type: str, properties: dict[str, Any]
|
|
944
|
+
) -> dict[str, Any]:
|
|
944
945
|
"""Return provider equivalent keyword names from EODAG keywords.
|
|
945
946
|
|
|
946
947
|
:param product_type: product type id
|
|
@@ -973,12 +974,12 @@ class ECMWFSearch(PostJsonSearch):
|
|
|
973
974
|
if hasattr(self, "auth") and isinstance(self.auth, AuthBase)
|
|
974
975
|
else None
|
|
975
976
|
)
|
|
976
|
-
timeout = getattr(self.config, "timeout",
|
|
977
|
-
return fetch_json(url, auth=auth, timeout=timeout)
|
|
977
|
+
timeout = getattr(self.config, "timeout", DEFAULT_SEARCH_TIMEOUT)
|
|
978
|
+
return functools.lru_cache()(fetch_json)(url, auth=auth, timeout=timeout)
|
|
978
979
|
|
|
979
980
|
def normalize_results(
|
|
980
981
|
self, results: RawSearchResult, **kwargs: Any
|
|
981
|
-
) ->
|
|
982
|
+
) -> list[EOProduct]:
|
|
982
983
|
"""Build :class:`~eodag.api.product._product.EOProduct` from provider result
|
|
983
984
|
|
|
984
985
|
:param results: Raw provider result as single dict in list
|
|
@@ -1150,7 +1151,7 @@ class MeteoblueSearch(ECMWFSearch):
|
|
|
1150
1151
|
self,
|
|
1151
1152
|
prep: PreparedSearch = PreparedSearch(),
|
|
1152
1153
|
**kwargs: Any,
|
|
1153
|
-
) ->
|
|
1154
|
+
) -> tuple[list[str], int]:
|
|
1154
1155
|
"""Wraps PostJsonSearch.collect_search_urls to force product count to 1
|
|
1155
1156
|
|
|
1156
1157
|
:param prep: :class:`~eodag.plugins.search.PreparedSearch` object containing information for the search
|
|
@@ -1162,7 +1163,7 @@ class MeteoblueSearch(ECMWFSearch):
|
|
|
1162
1163
|
|
|
1163
1164
|
def do_search(
|
|
1164
1165
|
self, prep: PreparedSearch = PreparedSearch(items_per_page=None), **kwargs: Any
|
|
1165
|
-
) ->
|
|
1166
|
+
) -> list[dict[str, Any]]:
|
|
1166
1167
|
"""Perform the actual search request, and return result in a single element.
|
|
1167
1168
|
|
|
1168
1169
|
:param prep: :class:`~eodag.plugins.search.PreparedSearch` object containing information for the search
|
|
@@ -1182,7 +1183,7 @@ class MeteoblueSearch(ECMWFSearch):
|
|
|
1182
1183
|
|
|
1183
1184
|
def build_query_string(
|
|
1184
1185
|
self, product_type: str, **kwargs: Any
|
|
1185
|
-
) ->
|
|
1186
|
+
) -> tuple[dict[str, Any], str]:
|
|
1186
1187
|
"""Build The query string using the search parameters
|
|
1187
1188
|
|
|
1188
1189
|
:param product_type: product type id
|
|
@@ -1221,7 +1222,7 @@ class WekeoECMWFSearch(ECMWFSearch):
|
|
|
1221
1222
|
|
|
1222
1223
|
def normalize_results(
|
|
1223
1224
|
self, results: RawSearchResult, **kwargs: Any
|
|
1224
|
-
) ->
|
|
1225
|
+
) -> list[EOProduct]:
|
|
1225
1226
|
"""Build :class:`~eodag.api.product._product.EOProduct` from provider result
|
|
1226
1227
|
|
|
1227
1228
|
:param results: Raw provider result as single dict in list
|
|
@@ -1247,7 +1248,7 @@ class WekeoECMWFSearch(ECMWFSearch):
|
|
|
1247
1248
|
|
|
1248
1249
|
return normalized
|
|
1249
1250
|
|
|
1250
|
-
def do_search(self, *args: Any, **kwargs: Any) ->
|
|
1251
|
+
def do_search(self, *args: Any, **kwargs: Any) -> list[dict[str, Any]]:
|
|
1251
1252
|
"""Should perform the actual search request.
|
|
1252
1253
|
|
|
1253
1254
|
:param args: arguments to be used in the search
|
|
@@ -1258,7 +1259,7 @@ class WekeoECMWFSearch(ECMWFSearch):
|
|
|
1258
1259
|
|
|
1259
1260
|
def build_query_string(
|
|
1260
1261
|
self, product_type: str, **kwargs: Any
|
|
1261
|
-
) ->
|
|
1262
|
+
) -> tuple[dict[str, Any], str]:
|
|
1262
1263
|
"""Build The query string using the search parameters
|
|
1263
1264
|
|
|
1264
1265
|
:param product_type: product type id
|
|
@@ -22,7 +22,7 @@ import logging
|
|
|
22
22
|
import os
|
|
23
23
|
import re
|
|
24
24
|
from datetime import datetime
|
|
25
|
-
from typing import TYPE_CHECKING, Any,
|
|
25
|
+
from typing import TYPE_CHECKING, Any, Optional, cast
|
|
26
26
|
from urllib.parse import urlsplit
|
|
27
27
|
|
|
28
28
|
import boto3
|
|
@@ -69,8 +69,8 @@ def _get_date_from_yyyymmdd(date_str: str, item_key: str) -> Optional[datetime]:
|
|
|
69
69
|
|
|
70
70
|
|
|
71
71
|
def _get_dates_from_dataset_data(
|
|
72
|
-
dataset_item:
|
|
73
|
-
) -> Optional[
|
|
72
|
+
dataset_item: dict[str, Any]
|
|
73
|
+
) -> Optional[dict[str, str]]:
|
|
74
74
|
dates = {}
|
|
75
75
|
if "start_datetime" in dataset_item["properties"]:
|
|
76
76
|
dates["start"] = dataset_item["properties"]["start_datetime"]
|
|
@@ -96,7 +96,7 @@ def _get_s3_client(endpoint_url: str) -> S3Client:
|
|
|
96
96
|
)
|
|
97
97
|
|
|
98
98
|
|
|
99
|
-
def _check_int_values_properties(properties:
|
|
99
|
+
def _check_int_values_properties(properties: dict[str, Any]):
|
|
100
100
|
# remove int values with a bit length of more than 64 from the properties
|
|
101
101
|
invalid = []
|
|
102
102
|
for prop, prop_value in properties.items():
|
|
@@ -134,7 +134,7 @@ class CopMarineSearch(StaticStacSearch):
|
|
|
134
134
|
|
|
135
135
|
def _get_product_type_info(
|
|
136
136
|
self, product_type: str
|
|
137
|
-
) ->
|
|
137
|
+
) -> tuple[dict[str, Any], list[dict[str, Any]]]:
|
|
138
138
|
"""Fetch product type and associated datasets info"""
|
|
139
139
|
|
|
140
140
|
fetch_url = cast(str, self.config.discover_product_types["fetch_url"]).format(
|
|
@@ -183,13 +183,23 @@ class CopMarineSearch(StaticStacSearch):
|
|
|
183
183
|
product_id: str,
|
|
184
184
|
s3_url: str,
|
|
185
185
|
product_type: str,
|
|
186
|
-
dataset_item:
|
|
187
|
-
collection_dict:
|
|
186
|
+
dataset_item: dict[str, Any],
|
|
187
|
+
collection_dict: dict[str, Any],
|
|
188
188
|
):
|
|
189
|
+
# try to find date(s) in product id
|
|
190
|
+
item_dates = re.findall(r"(\d{4})(0[1-9]|1[0-2])([0-3]\d)", product_id)
|
|
191
|
+
if not item_dates:
|
|
192
|
+
item_dates = re.findall(r"_(\d{4})(0[1-9]|1[0-2])", product_id)
|
|
193
|
+
use_dataset_dates = not bool(item_dates)
|
|
189
194
|
for obj in collection_objects["Contents"]:
|
|
190
195
|
if product_id in obj["Key"]:
|
|
191
196
|
return self._create_product(
|
|
192
|
-
product_type,
|
|
197
|
+
product_type,
|
|
198
|
+
obj["Key"],
|
|
199
|
+
s3_url,
|
|
200
|
+
dataset_item,
|
|
201
|
+
collection_dict,
|
|
202
|
+
use_dataset_dates,
|
|
193
203
|
)
|
|
194
204
|
return None
|
|
195
205
|
|
|
@@ -198,8 +208,8 @@ class CopMarineSearch(StaticStacSearch):
|
|
|
198
208
|
product_type: str,
|
|
199
209
|
item_key: str,
|
|
200
210
|
s3_url: str,
|
|
201
|
-
dataset_item:
|
|
202
|
-
collection_dict:
|
|
211
|
+
dataset_item: dict[str, Any],
|
|
212
|
+
collection_dict: dict[str, Any],
|
|
203
213
|
use_dataset_dates: bool = False,
|
|
204
214
|
) -> Optional[EOProduct]:
|
|
205
215
|
|
|
@@ -278,7 +288,7 @@ class CopMarineSearch(StaticStacSearch):
|
|
|
278
288
|
self,
|
|
279
289
|
prep: PreparedSearch = PreparedSearch(),
|
|
280
290
|
**kwargs: Any,
|
|
281
|
-
) ->
|
|
291
|
+
) -> tuple[list[EOProduct], Optional[int]]:
|
|
282
292
|
"""
|
|
283
293
|
Implementation of search for the Copernicus Marine provider
|
|
284
294
|
:param prep: object containing search parameterds
|
|
@@ -298,7 +308,7 @@ class CopMarineSearch(StaticStacSearch):
|
|
|
298
308
|
"parameter product type is required for search with cop_marine provider"
|
|
299
309
|
)
|
|
300
310
|
collection_dict, datasets_items_list = self._get_product_type_info(product_type)
|
|
301
|
-
products:
|
|
311
|
+
products: list[EOProduct] = []
|
|
302
312
|
start_index = items_per_page * (page - 1) + 1
|
|
303
313
|
num_total = 0
|
|
304
314
|
for i, dataset_item in enumerate(datasets_items_list):
|
|
@@ -17,26 +17,16 @@
|
|
|
17
17
|
# limitations under the License.
|
|
18
18
|
import logging
|
|
19
19
|
from types import MethodType
|
|
20
|
-
from typing import Any
|
|
20
|
+
from typing import Any
|
|
21
21
|
|
|
22
|
-
import boto3
|
|
23
|
-
import botocore
|
|
24
22
|
from botocore.exceptions import BotoCoreError
|
|
25
23
|
|
|
26
|
-
from eodag.api.product import
|
|
24
|
+
from eodag.api.product import EOProduct # type: ignore
|
|
27
25
|
from eodag.api.search_result import RawSearchResult
|
|
28
|
-
from eodag.config import PluginConfig
|
|
29
|
-
from eodag.plugins.authentication.aws_auth import AwsAuth
|
|
30
26
|
from eodag.plugins.search.qssearch import ODataV4Search
|
|
31
|
-
from eodag.utils import
|
|
32
|
-
from eodag.utils.
|
|
33
|
-
AuthenticationError,
|
|
34
|
-
MisconfiguredError,
|
|
35
|
-
NotAvailableError,
|
|
36
|
-
RequestError,
|
|
37
|
-
)
|
|
27
|
+
from eodag.utils.exceptions import RequestError
|
|
28
|
+
from eodag.utils.s3 import update_assets_from_s3
|
|
38
29
|
|
|
39
|
-
DATA_EXTENSIONS = ["jp2", "tiff", "nc", "grib"]
|
|
40
30
|
logger = logging.getLogger("eodag.search.creodiass3")
|
|
41
31
|
|
|
42
32
|
|
|
@@ -54,73 +44,13 @@ def patched_register_downloader(self, downloader, authenticator):
|
|
|
54
44
|
self.register_downloader_only(downloader, authenticator)
|
|
55
45
|
# and also update assets
|
|
56
46
|
try:
|
|
57
|
-
|
|
47
|
+
update_assets_from_s3(
|
|
48
|
+
self, authenticator, getattr(downloader.config, "s3_endpoint", None)
|
|
49
|
+
)
|
|
58
50
|
except BotoCoreError as e:
|
|
59
51
|
raise RequestError.from_error(e, "could not update assets") from e
|
|
60
52
|
|
|
61
53
|
|
|
62
|
-
def _update_assets(product: EOProduct, config: PluginConfig, auth: AwsAuth):
|
|
63
|
-
product.assets = AssetsDict(product)
|
|
64
|
-
prefix = (
|
|
65
|
-
product.properties.get("productIdentifier", None).replace("/eodata/", "") + "/"
|
|
66
|
-
)
|
|
67
|
-
if prefix:
|
|
68
|
-
try:
|
|
69
|
-
auth_dict = auth.authenticate()
|
|
70
|
-
required_creds = ["aws_access_key_id", "aws_secret_access_key"]
|
|
71
|
-
if not all(x in auth_dict for x in required_creds):
|
|
72
|
-
raise MisconfiguredError(
|
|
73
|
-
f"Incomplete credentials for {product.provider}, missing "
|
|
74
|
-
f"{[x for x in required_creds if x not in auth_dict]}"
|
|
75
|
-
)
|
|
76
|
-
if not getattr(auth, "s3_client", None):
|
|
77
|
-
auth.s3_client = boto3.client(
|
|
78
|
-
"s3",
|
|
79
|
-
endpoint_url=config.s3_endpoint,
|
|
80
|
-
aws_access_key_id=auth_dict["aws_access_key_id"],
|
|
81
|
-
aws_secret_access_key=auth_dict["aws_secret_access_key"],
|
|
82
|
-
)
|
|
83
|
-
logger.debug("Listing assets in %s", prefix)
|
|
84
|
-
product.assets = AssetsDict(product)
|
|
85
|
-
s3_res = auth.s3_client.list_objects(
|
|
86
|
-
Bucket=config.s3_bucket, Prefix=prefix, MaxKeys=300
|
|
87
|
-
)
|
|
88
|
-
# check if product path has assets or is already a file
|
|
89
|
-
if "Contents" in s3_res:
|
|
90
|
-
for asset in s3_res["Contents"]:
|
|
91
|
-
asset_basename = (
|
|
92
|
-
asset["Key"].split("/")[-1]
|
|
93
|
-
if "/" in asset["Key"]
|
|
94
|
-
else asset["Key"]
|
|
95
|
-
)
|
|
96
|
-
|
|
97
|
-
if len(asset_basename) > 0 and asset_basename not in product.assets:
|
|
98
|
-
role = (
|
|
99
|
-
"data"
|
|
100
|
-
if asset_basename.split(".")[-1] in DATA_EXTENSIONS
|
|
101
|
-
else "metadata"
|
|
102
|
-
)
|
|
103
|
-
|
|
104
|
-
product.assets[asset_basename] = {
|
|
105
|
-
"title": asset_basename,
|
|
106
|
-
"roles": [role],
|
|
107
|
-
"href": f"s3://{config.s3_bucket}/{asset['Key']}",
|
|
108
|
-
}
|
|
109
|
-
if mime_type := guess_file_type(asset["Key"]):
|
|
110
|
-
product.assets[asset_basename]["type"] = mime_type
|
|
111
|
-
# update driver
|
|
112
|
-
product.driver = product.get_driver()
|
|
113
|
-
|
|
114
|
-
except botocore.exceptions.ClientError as e:
|
|
115
|
-
if str(auth.config.auth_error_code) in str(e):
|
|
116
|
-
raise AuthenticationError(
|
|
117
|
-
f"Authentication failed on {config.base_uri} s3"
|
|
118
|
-
) from e
|
|
119
|
-
raise NotAvailableError(
|
|
120
|
-
f"assets for product {prefix} could not be found"
|
|
121
|
-
) from e
|
|
122
|
-
|
|
123
|
-
|
|
124
54
|
class CreodiasS3Search(ODataV4Search):
|
|
125
55
|
"""
|
|
126
56
|
``CreodiasS3Search`` is an extension of :class:`~eodag.plugins.search.qssearch.ODataV4Search`,
|
|
@@ -139,7 +69,7 @@ class CreodiasS3Search(ODataV4Search):
|
|
|
139
69
|
|
|
140
70
|
def normalize_results(
|
|
141
71
|
self, results: RawSearchResult, **kwargs: Any
|
|
142
|
-
) ->
|
|
72
|
+
) -> list[EOProduct]:
|
|
143
73
|
"""Build EOProducts from provider results"""
|
|
144
74
|
|
|
145
75
|
products = super(CreodiasS3Search, self).normalize_results(results, **kwargs)
|
eodag/plugins/search/csw.py
CHANGED
|
@@ -19,7 +19,7 @@ from __future__ import annotations
|
|
|
19
19
|
|
|
20
20
|
import logging
|
|
21
21
|
import re
|
|
22
|
-
from typing import TYPE_CHECKING, Any,
|
|
22
|
+
from typing import TYPE_CHECKING, Any, Optional, Union
|
|
23
23
|
|
|
24
24
|
import pyproj
|
|
25
25
|
from owslib.csw import CatalogueServiceWeb
|
|
@@ -60,13 +60,13 @@ class CSWSearch(Search):
|
|
|
60
60
|
* :attr:`~eodag.config.PluginConfig.api_endpoint` (``str``) (**mandatory**): The endpoint of the
|
|
61
61
|
provider's search interface
|
|
62
62
|
* :attr:`~eodag.config.PluginConfig.version` (``str``): OGC Catalogue Service version; default: ``2.0.2``
|
|
63
|
-
* :attr:`~eodag.config.PluginConfig.search_definition` (``
|
|
63
|
+
* :attr:`~eodag.config.PluginConfig.search_definition` (``dict[str, Any]``) (**mandatory**):
|
|
64
64
|
|
|
65
|
-
* **product_type_tags** (``
|
|
65
|
+
* **product_type_tags** (``list[dict[str, Any]``): dict of product type tags
|
|
66
66
|
* **resource_location_filter** (``str``): regex string
|
|
67
|
-
* **date_tags** (``
|
|
67
|
+
* **date_tags** (``dict[str, Any]``): tags for start and end
|
|
68
68
|
|
|
69
|
-
* :attr:`~eodag.config.PluginConfig.metadata_mapping` (``
|
|
69
|
+
* :attr:`~eodag.config.PluginConfig.metadata_mapping` (``dict[str, Any]``): The search plugins of this kind can
|
|
70
70
|
detect when a metadata mapping is "query-able", and get the semantics of how to format the query string
|
|
71
71
|
parameter that enables to make a query on the corresponding metadata. To make a metadata query-able,
|
|
72
72
|
just configure it in the metadata mapping to be a list of 2 items, the first one being the
|
|
@@ -107,7 +107,7 @@ class CSWSearch(Search):
|
|
|
107
107
|
self,
|
|
108
108
|
prep: PreparedSearch = PreparedSearch(),
|
|
109
109
|
**kwargs: Any,
|
|
110
|
-
) ->
|
|
110
|
+
) -> tuple[list[EOProduct], Optional[int]]:
|
|
111
111
|
"""Perform a search on a OGC/CSW-like interface"""
|
|
112
112
|
product_type = kwargs.get("productType")
|
|
113
113
|
if product_type is None:
|
|
@@ -117,7 +117,7 @@ class CSWSearch(Search):
|
|
|
117
117
|
self.__init_catalog(**getattr(auth.config, "credentials", {}))
|
|
118
118
|
else:
|
|
119
119
|
self.__init_catalog()
|
|
120
|
-
results:
|
|
120
|
+
results: list[EOProduct] = []
|
|
121
121
|
if self.catalog:
|
|
122
122
|
provider_product_type = self.config.products[product_type]["productType"]
|
|
123
123
|
for product_type_def in self.config.search_definition["product_type_tags"]:
|
|
@@ -229,12 +229,12 @@ class CSWSearch(Search):
|
|
|
229
229
|
|
|
230
230
|
def __convert_query_params(
|
|
231
231
|
self,
|
|
232
|
-
product_type_def:
|
|
232
|
+
product_type_def: dict[str, Any],
|
|
233
233
|
product_type: str,
|
|
234
|
-
params:
|
|
235
|
-
) -> Union[
|
|
234
|
+
params: dict[str, Any],
|
|
235
|
+
) -> Union[list[OgcExpression], list[list[OgcExpression]]]:
|
|
236
236
|
"""Translates eodag search to CSW constraints using owslib constraint classes"""
|
|
237
|
-
constraints:
|
|
237
|
+
constraints: list[OgcExpression] = []
|
|
238
238
|
# How the match should be performed (fuzzy, prefix, postfix or exact).
|
|
239
239
|
# defaults to fuzzy
|
|
240
240
|
pt_tag, matching = (
|