eodag 3.1.0b2__py3-none-any.whl → 3.2.1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- eodag/api/core.py +10 -11
- eodag/api/product/_assets.py +45 -9
- eodag/api/product/_product.py +14 -18
- eodag/api/product/metadata_mapping.py +23 -5
- eodag/config.py +11 -11
- eodag/plugins/apis/ecmwf.py +2 -6
- eodag/plugins/apis/usgs.py +1 -1
- eodag/plugins/authentication/openid_connect.py +6 -0
- eodag/plugins/download/aws.py +90 -11
- eodag/plugins/search/base.py +3 -2
- eodag/plugins/search/build_search_result.py +348 -281
- eodag/plugins/search/data_request_search.py +3 -3
- eodag/plugins/search/qssearch.py +32 -103
- eodag/plugins/search/static_stac_search.py +1 -1
- eodag/resources/ext_product_types.json +1 -1
- eodag/resources/product_types.yml +564 -114
- eodag/resources/providers.yml +956 -1173
- eodag/resources/user_conf_template.yml +1 -11
- eodag/rest/stac.py +1 -0
- eodag/rest/types/queryables.py +28 -16
- eodag/types/__init__.py +73 -11
- eodag/utils/__init__.py +2 -2
- eodag/utils/s3.py +31 -8
- {eodag-3.1.0b2.dist-info → eodag-3.2.1.dist-info}/METADATA +10 -9
- {eodag-3.1.0b2.dist-info → eodag-3.2.1.dist-info}/RECORD +29 -29
- {eodag-3.1.0b2.dist-info → eodag-3.2.1.dist-info}/WHEEL +1 -1
- {eodag-3.1.0b2.dist-info → eodag-3.2.1.dist-info}/entry_points.txt +0 -0
- {eodag-3.1.0b2.dist-info → eodag-3.2.1.dist-info/licenses}/LICENSE +0 -0
- {eodag-3.1.0b2.dist-info → eodag-3.2.1.dist-info}/top_level.txt +0 -0
eodag/api/core.py
CHANGED
|
@@ -23,13 +23,13 @@ import os
|
|
|
23
23
|
import re
|
|
24
24
|
import shutil
|
|
25
25
|
import tempfile
|
|
26
|
+
from importlib.metadata import version
|
|
27
|
+
from importlib.resources import files as res_files
|
|
26
28
|
from operator import itemgetter
|
|
27
29
|
from typing import TYPE_CHECKING, Any, Iterator, Optional, Union
|
|
28
30
|
|
|
29
31
|
import geojson
|
|
30
|
-
import pkg_resources
|
|
31
32
|
import yaml.parser
|
|
32
|
-
from pkg_resources import resource_filename
|
|
33
33
|
from whoosh import analysis, fields
|
|
34
34
|
from whoosh.fields import Schema
|
|
35
35
|
from whoosh.index import exists_in, open_dir
|
|
@@ -119,8 +119,8 @@ class EODataAccessGateway:
|
|
|
119
119
|
user_conf_file_path: Optional[str] = None,
|
|
120
120
|
locations_conf_path: Optional[str] = None,
|
|
121
121
|
) -> None:
|
|
122
|
-
product_types_config_path =
|
|
123
|
-
"eodag"
|
|
122
|
+
product_types_config_path = os.getenv("EODAG_PRODUCT_TYPES_CFG_FILE") or str(
|
|
123
|
+
res_files("eodag") / "resources" / "product_types.yml"
|
|
124
124
|
)
|
|
125
125
|
self.product_types_config = SimpleYamlProxyConfig(product_types_config_path)
|
|
126
126
|
self.product_types_config_md5 = obj_md5sum(self.product_types_config.source)
|
|
@@ -161,8 +161,8 @@ class EODataAccessGateway:
|
|
|
161
161
|
user_conf_file_path = standard_configuration_path
|
|
162
162
|
if not os.path.isfile(standard_configuration_path):
|
|
163
163
|
shutil.copy(
|
|
164
|
-
|
|
165
|
-
"eodag"
|
|
164
|
+
str(
|
|
165
|
+
res_files("eodag") / "resources" / "user_conf_template.yml"
|
|
166
166
|
),
|
|
167
167
|
standard_configuration_path,
|
|
168
168
|
)
|
|
@@ -203,9 +203,8 @@ class EODataAccessGateway:
|
|
|
203
203
|
locations_conf_path = os.path.join(self.conf_dir, "locations.yml")
|
|
204
204
|
if not os.path.isfile(locations_conf_path):
|
|
205
205
|
# copy locations conf file and replace path example
|
|
206
|
-
locations_conf_template =
|
|
207
|
-
"eodag"
|
|
208
|
-
os.path.join("resources", "locations_conf_template.yml"),
|
|
206
|
+
locations_conf_template = str(
|
|
207
|
+
res_files("eodag") / "resources" / "locations_conf_template.yml"
|
|
209
208
|
)
|
|
210
209
|
with (
|
|
211
210
|
open(locations_conf_template) as infile,
|
|
@@ -222,14 +221,14 @@ class EODataAccessGateway:
|
|
|
222
221
|
outfile.write(line)
|
|
223
222
|
# copy sample shapefile dir
|
|
224
223
|
shutil.copytree(
|
|
225
|
-
|
|
224
|
+
str(res_files("eodag") / "resources" / "shp"),
|
|
226
225
|
os.path.join(self.conf_dir, "shp"),
|
|
227
226
|
)
|
|
228
227
|
self.set_locations_conf(locations_conf_path)
|
|
229
228
|
|
|
230
229
|
def get_version(self) -> str:
|
|
231
230
|
"""Get eodag package version"""
|
|
232
|
-
return
|
|
231
|
+
return version("eodag")
|
|
233
232
|
|
|
234
233
|
def build_index(self) -> None:
|
|
235
234
|
"""Build a `Whoosh <https://whoosh.readthedocs.io/en/latest/index.html>`_
|
eodag/api/product/_assets.py
CHANGED
|
@@ -31,12 +31,27 @@ if TYPE_CHECKING:
|
|
|
31
31
|
|
|
32
32
|
|
|
33
33
|
class AssetsDict(UserDict):
|
|
34
|
-
"""A UserDict object
|
|
35
|
-
:class:`~eodag.api.product._product.EOProduct` resulting from a
|
|
34
|
+
"""A UserDict object which values are :class:`~eodag.api.product._assets.Asset`
|
|
35
|
+
contained in a :class:`~eodag.api.product._product.EOProduct` resulting from a
|
|
36
|
+
search.
|
|
36
37
|
|
|
37
38
|
:param product: Product resulting from a search
|
|
38
39
|
:param args: (optional) Arguments used to init the dictionary
|
|
39
40
|
:param kwargs: (optional) Additional named-arguments used to init the dictionary
|
|
41
|
+
|
|
42
|
+
Example
|
|
43
|
+
-------
|
|
44
|
+
|
|
45
|
+
>>> from eodag.api.product import EOProduct
|
|
46
|
+
>>> product = EOProduct(
|
|
47
|
+
... provider="foo",
|
|
48
|
+
... properties={"id": "bar", "geometry": "POINT (0 0)"}
|
|
49
|
+
... )
|
|
50
|
+
>>> type(product.assets)
|
|
51
|
+
<class 'eodag.api.product._assets.AssetsDict'>
|
|
52
|
+
>>> product.assets.update({"foo": {"href": "http://somewhere/something"}})
|
|
53
|
+
>>> product.assets
|
|
54
|
+
{'foo': {'href': 'http://somewhere/something'}}
|
|
40
55
|
"""
|
|
41
56
|
|
|
42
57
|
product: EOProduct
|
|
@@ -56,22 +71,29 @@ class AssetsDict(UserDict):
|
|
|
56
71
|
"""
|
|
57
72
|
return {k: v.as_dict() for k, v in self.data.items()}
|
|
58
73
|
|
|
59
|
-
def get_values(self, asset_filter: str = "") -> list[Asset]:
|
|
74
|
+
def get_values(self, asset_filter: str = "", regex=True) -> list[Asset]:
|
|
60
75
|
"""
|
|
61
76
|
retrieves the assets matching the given filter
|
|
62
77
|
|
|
63
78
|
:param asset_filter: regex filter with which the assets should be matched
|
|
79
|
+
:param regex: Uses regex to match the asset key or simply compare strings
|
|
64
80
|
:return: list of assets
|
|
65
81
|
"""
|
|
66
82
|
if asset_filter:
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
|
|
83
|
+
if regex:
|
|
84
|
+
filter_regex = re.compile(asset_filter)
|
|
85
|
+
assets_keys = list(self.keys())
|
|
86
|
+
assets_keys = list(filter(filter_regex.fullmatch, assets_keys))
|
|
87
|
+
else:
|
|
88
|
+
assets_keys = [a for a in self.keys() if a == asset_filter]
|
|
70
89
|
filtered_assets = {}
|
|
71
90
|
if len(assets_keys) > 0:
|
|
72
91
|
filtered_assets = {a_key: self.get(a_key) for a_key in assets_keys}
|
|
73
92
|
assets_values = [a for a in filtered_assets.values() if a and "href" in a]
|
|
74
|
-
if not assets_values:
|
|
93
|
+
if not assets_values and regex:
|
|
94
|
+
# retry without regex
|
|
95
|
+
return self.get_values(asset_filter, regex=False)
|
|
96
|
+
elif not assets_values:
|
|
75
97
|
raise NotAvailableError(
|
|
76
98
|
rf"No asset key matching re.fullmatch(r'{asset_filter}') was found in {self.product}"
|
|
77
99
|
)
|
|
@@ -119,13 +141,27 @@ class AssetsDict(UserDict):
|
|
|
119
141
|
|
|
120
142
|
|
|
121
143
|
class Asset(UserDict):
|
|
122
|
-
"""A UserDict object containg one of the
|
|
123
|
-
:
|
|
144
|
+
"""A UserDict object containg one of the
|
|
145
|
+
:attr:`~eodag.api.product._product.EOProduct.assets` resulting from a search.
|
|
124
146
|
|
|
125
147
|
:param product: Product resulting from a search
|
|
126
148
|
:param key: asset key
|
|
127
149
|
:param args: (optional) Arguments used to init the dictionary
|
|
128
150
|
:param kwargs: (optional) Additional named-arguments used to init the dictionary
|
|
151
|
+
|
|
152
|
+
Example
|
|
153
|
+
-------
|
|
154
|
+
|
|
155
|
+
>>> from eodag.api.product import EOProduct
|
|
156
|
+
>>> product = EOProduct(
|
|
157
|
+
... provider="foo",
|
|
158
|
+
... properties={"id": "bar", "geometry": "POINT (0 0)"}
|
|
159
|
+
... )
|
|
160
|
+
>>> product.assets.update({"foo": {"href": "http://somewhere/something"}})
|
|
161
|
+
>>> type(product.assets["foo"])
|
|
162
|
+
<class 'eodag.api.product._assets.Asset'>
|
|
163
|
+
>>> product.assets["foo"]
|
|
164
|
+
{'href': 'http://somewhere/something'}
|
|
129
165
|
"""
|
|
130
166
|
|
|
131
167
|
product: EOProduct
|
eodag/api/product/_product.py
CHANGED
|
@@ -89,20 +89,6 @@ class EOProduct:
|
|
|
89
89
|
|
|
90
90
|
:param provider: The provider from which the product originates
|
|
91
91
|
:param properties: The metadata of the product
|
|
92
|
-
:ivar product_type: The product type
|
|
93
|
-
:vartype product_type: str
|
|
94
|
-
:ivar location: The path to the product, either remote or local if downloaded
|
|
95
|
-
:vartype location: str
|
|
96
|
-
:ivar remote_location: The remote path to the product
|
|
97
|
-
:vartype remote_location: str
|
|
98
|
-
:ivar search_kwargs: The search kwargs used by eodag to search for the product
|
|
99
|
-
:vartype search_kwargs: Any
|
|
100
|
-
:ivar geometry: The geometry of the product
|
|
101
|
-
:vartype geometry: :class:`shapely.geometry.base.BaseGeometry`
|
|
102
|
-
:ivar search_intersection: The intersection between the product's geometry
|
|
103
|
-
and the search area.
|
|
104
|
-
:vartype search_intersection: :class:`shapely.geometry.base.BaseGeometry` or None
|
|
105
|
-
|
|
106
92
|
|
|
107
93
|
.. note::
|
|
108
94
|
The geojson spec `enforces <https://github.com/geojson/draft-geojson/pull/6>`_
|
|
@@ -112,18 +98,28 @@ class EOProduct:
|
|
|
112
98
|
mentioned CRS.
|
|
113
99
|
"""
|
|
114
100
|
|
|
101
|
+
#: The provider from which the product originates
|
|
115
102
|
provider: str
|
|
103
|
+
#: The metadata of the product
|
|
116
104
|
properties: dict[str, Any]
|
|
105
|
+
#: The product type
|
|
117
106
|
product_type: Optional[str]
|
|
118
|
-
|
|
119
|
-
filename: str
|
|
120
|
-
remote_location: str
|
|
121
|
-
search_kwargs: Any
|
|
107
|
+
#: The geometry of the product
|
|
122
108
|
geometry: BaseGeometry
|
|
109
|
+
#: The intersection between the product's geometry and the search area.
|
|
123
110
|
search_intersection: Optional[BaseGeometry]
|
|
111
|
+
#: The path to the product, either remote or local if downloaded
|
|
112
|
+
location: str
|
|
113
|
+
#: The remote path to the product
|
|
114
|
+
remote_location: str
|
|
115
|
+
#: Assets of the product
|
|
124
116
|
assets: AssetsDict
|
|
125
117
|
#: Driver enables additional methods to be called on the EOProduct
|
|
126
118
|
driver: DatasetDriver
|
|
119
|
+
#: Product data filename, stored during download
|
|
120
|
+
filename: str
|
|
121
|
+
#: Product search keyword arguments, stored during search
|
|
122
|
+
search_kwargs: Any
|
|
127
123
|
|
|
128
124
|
def __init__(
|
|
129
125
|
self, provider: str, properties: dict[str, Any], **kwargs: Any
|
|
@@ -49,6 +49,7 @@ from eodag.utils import (
|
|
|
49
49
|
get_timestamp,
|
|
50
50
|
items_recursive_apply,
|
|
51
51
|
nested_pairs2dict,
|
|
52
|
+
sanitize,
|
|
52
53
|
string_to_jsonpath,
|
|
53
54
|
update_nested_dict,
|
|
54
55
|
)
|
|
@@ -176,6 +177,7 @@ def format_metadata(search_param: str, *args: Any, **kwargs: Any) -> str:
|
|
|
176
177
|
- ``split_corine_id``: get the product type by splitting the product id
|
|
177
178
|
- ``to_datetime_dict``: convert a datetime string to a dictionary where values are either a string or a list
|
|
178
179
|
- ``get_ecmwf_time``: get the time of a datetime string in the ECMWF format
|
|
180
|
+
- ``sanitize``: sanitize string
|
|
179
181
|
|
|
180
182
|
:param search_param: The string to be formatted
|
|
181
183
|
:param args: (optional) Additional arguments to use in the formatting process
|
|
@@ -367,8 +369,8 @@ def format_metadata(search_param: str, *args: Any, **kwargs: Any) -> str:
|
|
|
367
369
|
)
|
|
368
370
|
|
|
369
371
|
@staticmethod
|
|
370
|
-
def convert_to_geojson(
|
|
371
|
-
return geojson.dumps(
|
|
372
|
+
def convert_to_geojson(value: Any) -> str:
|
|
373
|
+
return geojson.dumps(value)
|
|
372
374
|
|
|
373
375
|
@staticmethod
|
|
374
376
|
def convert_from_ewkt(ewkt_string: str) -> Union[BaseGeometry, str]:
|
|
@@ -496,9 +498,16 @@ def format_metadata(search_param: str, *args: Any, **kwargs: Any) -> str:
|
|
|
496
498
|
return NOT_AVAILABLE
|
|
497
499
|
|
|
498
500
|
@staticmethod
|
|
499
|
-
def convert_replace_str(
|
|
501
|
+
def convert_replace_str(value: Any, args: str) -> str:
|
|
502
|
+
if isinstance(value, dict):
|
|
503
|
+
value = MetadataFormatter.convert_to_geojson(value)
|
|
504
|
+
elif not isinstance(value, str):
|
|
505
|
+
raise TypeError(
|
|
506
|
+
f"convert_replace_str expects a string or a dict (apply to_geojson). Got {type(value)}"
|
|
507
|
+
)
|
|
508
|
+
|
|
500
509
|
old, new = ast.literal_eval(args)
|
|
501
|
-
return re.sub(old, new,
|
|
510
|
+
return re.sub(old, new, value)
|
|
502
511
|
|
|
503
512
|
@staticmethod
|
|
504
513
|
def convert_recursive_sub_str(
|
|
@@ -816,6 +825,11 @@ def format_metadata(search_param: str, *args: Any, **kwargs: Any) -> str:
|
|
|
816
825
|
+ ":00"
|
|
817
826
|
]
|
|
818
827
|
|
|
828
|
+
@staticmethod
|
|
829
|
+
def convert_sanitize(text: str) -> str:
|
|
830
|
+
"""Sanitize string"""
|
|
831
|
+
return sanitize(text)
|
|
832
|
+
|
|
819
833
|
@staticmethod
|
|
820
834
|
def convert_get_dates_from_string(text: str, split_param="-"):
|
|
821
835
|
reg = "[0-9]{8}" + split_param + "[0-9]{8}"
|
|
@@ -1487,7 +1501,11 @@ def get_queryable_from_provider(
|
|
|
1487
1501
|
ind = mapping_values.index(provider_queryable)
|
|
1488
1502
|
return Queryables.get_queryable_from_alias(list(metadata_mapping.keys())[ind])
|
|
1489
1503
|
for param, param_conf in metadata_mapping.items():
|
|
1490
|
-
if
|
|
1504
|
+
if (
|
|
1505
|
+
isinstance(param_conf, list)
|
|
1506
|
+
and param_conf[0]
|
|
1507
|
+
and re.search(pattern, param_conf[0])
|
|
1508
|
+
):
|
|
1491
1509
|
return Queryables.get_queryable_from_alias(param)
|
|
1492
1510
|
return None
|
|
1493
1511
|
|
eodag/config.py
CHANGED
|
@@ -20,6 +20,7 @@ from __future__ import annotations
|
|
|
20
20
|
import logging
|
|
21
21
|
import os
|
|
22
22
|
import tempfile
|
|
23
|
+
from importlib.resources import files as res_files
|
|
23
24
|
from inspect import isclass
|
|
24
25
|
from typing import (
|
|
25
26
|
Annotated,
|
|
@@ -41,7 +42,6 @@ import yaml.constructor
|
|
|
41
42
|
import yaml.parser
|
|
42
43
|
from annotated_types import Gt
|
|
43
44
|
from jsonpath_ng import JSONPath
|
|
44
|
-
from pkg_resources import resource_filename
|
|
45
45
|
|
|
46
46
|
from eodag.api.product.metadata_mapping import mtd_cfg_as_conversion_and_querypath
|
|
47
47
|
from eodag.utils import (
|
|
@@ -443,6 +443,9 @@ class PluginConfig(yaml.YAMLObject):
|
|
|
443
443
|
literal_search_params: dict[str, str]
|
|
444
444
|
#: :class:`~eodag.plugins.search.qssearch.QueryStringSearch` Characters that should not be quoted in the url params
|
|
445
445
|
dont_quote: list[str]
|
|
446
|
+
#: :class:`~eodag.plugins.search.qssearch.QueryStringSearch` Guess assets keys using their ``href``.
|
|
447
|
+
#: Use their original key if ``False``
|
|
448
|
+
asset_key_from_href: bool
|
|
446
449
|
#: :class:`~eodag.plugins.search.qssearch.ODataV4Search` Dict describing free text search request build
|
|
447
450
|
free_text_search_operations: dict[str, Any]
|
|
448
451
|
#: :class:`~eodag.plugins.search.qssearch.ODataV4Search` Set to ``True`` if the metadata is not given in the search
|
|
@@ -671,9 +674,9 @@ def load_default_config() -> dict[str, ProviderConfig]:
|
|
|
671
674
|
|
|
672
675
|
:returns: The default provider's configuration
|
|
673
676
|
"""
|
|
674
|
-
eodag_providers_cfg_file = os.getenv(
|
|
675
|
-
"
|
|
676
|
-
)
|
|
677
|
+
eodag_providers_cfg_file = os.getenv("EODAG_PROVIDERS_CFG_FILE") or str(
|
|
678
|
+
res_files("eodag") / "resources" / "providers.yml"
|
|
679
|
+
)
|
|
677
680
|
return load_config(eodag_providers_cfg_file)
|
|
678
681
|
|
|
679
682
|
|
|
@@ -779,6 +782,7 @@ def provider_config_init(
|
|
|
779
782
|
and provider_config.search.type
|
|
780
783
|
in [
|
|
781
784
|
"StacSearch",
|
|
785
|
+
"StacListAssets",
|
|
782
786
|
"StaticStacSearch",
|
|
783
787
|
]
|
|
784
788
|
):
|
|
@@ -987,9 +991,7 @@ def load_stac_config() -> dict[str, Any]:
|
|
|
987
991
|
|
|
988
992
|
:returns: The stac configuration
|
|
989
993
|
"""
|
|
990
|
-
return load_yml_config(
|
|
991
|
-
resource_filename("eodag", os.path.join("resources/", "stac.yml"))
|
|
992
|
-
)
|
|
994
|
+
return load_yml_config(str(res_files("eodag") / "resources" / "stac.yml"))
|
|
993
995
|
|
|
994
996
|
|
|
995
997
|
def load_stac_api_config() -> dict[str, Any]:
|
|
@@ -997,9 +999,7 @@ def load_stac_api_config() -> dict[str, Any]:
|
|
|
997
999
|
|
|
998
1000
|
:returns: The stac API configuration
|
|
999
1001
|
"""
|
|
1000
|
-
return load_yml_config(
|
|
1001
|
-
resource_filename("eodag", os.path.join("resources/", "stac_api.yml"))
|
|
1002
|
-
)
|
|
1002
|
+
return load_yml_config(str(res_files("eodag") / "resources" / "stac_api.yml"))
|
|
1003
1003
|
|
|
1004
1004
|
|
|
1005
1005
|
def load_stac_provider_config() -> dict[str, Any]:
|
|
@@ -1008,7 +1008,7 @@ def load_stac_provider_config() -> dict[str, Any]:
|
|
|
1008
1008
|
:returns: The stac provider configuration
|
|
1009
1009
|
"""
|
|
1010
1010
|
return SimpleYamlProxyConfig(
|
|
1011
|
-
|
|
1011
|
+
str(res_files("eodag") / "resources" / "stac_provider.yml")
|
|
1012
1012
|
).source
|
|
1013
1013
|
|
|
1014
1014
|
|
eodag/plugins/apis/ecmwf.py
CHANGED
|
@@ -30,11 +30,7 @@ from pydantic.fields import FieldInfo
|
|
|
30
30
|
from eodag.plugins.apis.base import Api
|
|
31
31
|
from eodag.plugins.search import PreparedSearch
|
|
32
32
|
from eodag.plugins.search.base import Search
|
|
33
|
-
from eodag.plugins.search.build_search_result import
|
|
34
|
-
ECMWF_KEYWORDS,
|
|
35
|
-
ECMWFSearch,
|
|
36
|
-
keywords_to_mdt,
|
|
37
|
-
)
|
|
33
|
+
from eodag.plugins.search.build_search_result import ECMWFSearch, ecmwf_mtd
|
|
38
34
|
from eodag.utils import (
|
|
39
35
|
DEFAULT_DOWNLOAD_TIMEOUT,
|
|
40
36
|
DEFAULT_DOWNLOAD_WAIT,
|
|
@@ -94,7 +90,7 @@ class EcmwfApi(Api, ECMWFSearch):
|
|
|
94
90
|
def __init__(self, provider: str, config: PluginConfig) -> None:
|
|
95
91
|
# init self.config.metadata_mapping using Search Base plugin
|
|
96
92
|
config.metadata_mapping = {
|
|
97
|
-
**
|
|
93
|
+
**ecmwf_mtd(),
|
|
98
94
|
**config.metadata_mapping,
|
|
99
95
|
}
|
|
100
96
|
Search.__init__(self, provider, config)
|
eodag/plugins/apis/usgs.py
CHANGED
|
@@ -379,6 +379,12 @@ class OIDCAuthorizationCodeFlowAuth(OIDCRefreshTokenBase):
|
|
|
379
379
|
|
|
380
380
|
login_document = etree.HTML(authorization_response.text)
|
|
381
381
|
login_forms = login_document.xpath(self.config.login_form_xpath)
|
|
382
|
+
|
|
383
|
+
if not login_forms:
|
|
384
|
+
# we assume user is already logged in
|
|
385
|
+
# no form found because we got redirected to the redirect_uri
|
|
386
|
+
return authorization_response
|
|
387
|
+
|
|
382
388
|
login_form = login_forms[0]
|
|
383
389
|
|
|
384
390
|
# Get the form data to pass to the login form from config or from the login form
|
eodag/plugins/download/aws.py
CHANGED
|
@@ -60,6 +60,7 @@ from eodag.utils.exceptions import (
|
|
|
60
60
|
NotAvailableError,
|
|
61
61
|
TimeOutError,
|
|
62
62
|
)
|
|
63
|
+
from eodag.utils.s3 import open_s3_zipped_object
|
|
63
64
|
|
|
64
65
|
if TYPE_CHECKING:
|
|
65
66
|
from boto3.resources.collection import ResourceCollection
|
|
@@ -195,6 +196,7 @@ AWS_AUTH_ERROR_MESSAGES = [
|
|
|
195
196
|
"AccessDenied",
|
|
196
197
|
"InvalidAccessKeyId",
|
|
197
198
|
"SignatureDoesNotMatch",
|
|
199
|
+
"InvalidRequest",
|
|
198
200
|
]
|
|
199
201
|
|
|
200
202
|
|
|
@@ -232,6 +234,7 @@ class AwsDownload(Download):
|
|
|
232
234
|
super(AwsDownload, self).__init__(provider, config)
|
|
233
235
|
self.requester_pays = getattr(self.config, "requester_pays", False)
|
|
234
236
|
self.s3_session: Optional[boto3.session.Session] = None
|
|
237
|
+
self.s3_resource: Optional[boto3.resources.base.ServiceResource] = None
|
|
235
238
|
|
|
236
239
|
def download(
|
|
237
240
|
self,
|
|
@@ -289,10 +292,10 @@ class AwsDownload(Download):
|
|
|
289
292
|
asset_filter = kwargs.get("asset", None)
|
|
290
293
|
if asset_filter:
|
|
291
294
|
build_safe = False
|
|
295
|
+
ignore_assets = False
|
|
292
296
|
else:
|
|
293
297
|
build_safe = product_conf.get("build_safe", False)
|
|
294
|
-
|
|
295
|
-
ignore_assets = getattr(self.config, "ignore_assets", False)
|
|
298
|
+
ignore_assets = getattr(self.config, "ignore_assets", False)
|
|
296
299
|
|
|
297
300
|
# product conf overrides provider conf for "flatten_top_dirs"
|
|
298
301
|
flatten_top_dirs = product_conf.get(
|
|
@@ -325,19 +328,32 @@ class AwsDownload(Download):
|
|
|
325
328
|
bucket_names_and_prefixes, auth
|
|
326
329
|
)
|
|
327
330
|
|
|
331
|
+
# files in zip
|
|
332
|
+
updated_bucket_names_and_prefixes = self._download_file_in_zip(
|
|
333
|
+
product, bucket_names_and_prefixes, product_local_path, progress_callback
|
|
334
|
+
)
|
|
335
|
+
# prevent nothing-to-download errors if download was performed in zip
|
|
336
|
+
raise_error = (
|
|
337
|
+
False
|
|
338
|
+
if len(updated_bucket_names_and_prefixes) != len(bucket_names_and_prefixes)
|
|
339
|
+
else True
|
|
340
|
+
)
|
|
341
|
+
|
|
328
342
|
# downloadable files
|
|
329
343
|
unique_product_chunks = self._get_unique_products(
|
|
330
|
-
|
|
344
|
+
updated_bucket_names_and_prefixes,
|
|
331
345
|
authenticated_objects,
|
|
332
346
|
asset_filter,
|
|
333
347
|
ignore_assets,
|
|
334
348
|
product,
|
|
349
|
+
raise_error=raise_error,
|
|
335
350
|
)
|
|
336
351
|
|
|
337
352
|
total_size = sum([p.size for p in unique_product_chunks]) or None
|
|
338
353
|
|
|
339
354
|
# download
|
|
340
|
-
|
|
355
|
+
if len(unique_product_chunks) > 0:
|
|
356
|
+
progress_callback.reset(total=total_size)
|
|
341
357
|
try:
|
|
342
358
|
for product_chunk in unique_product_chunks:
|
|
343
359
|
try:
|
|
@@ -389,6 +405,52 @@ class AwsDownload(Download):
|
|
|
389
405
|
|
|
390
406
|
return product_local_path
|
|
391
407
|
|
|
408
|
+
def _download_file_in_zip(
|
|
409
|
+
self, product, bucket_names_and_prefixes, product_local_path, progress_callback
|
|
410
|
+
):
|
|
411
|
+
"""
|
|
412
|
+
Download file in zip from a prefix like `foo/bar.zip!file.txt`
|
|
413
|
+
"""
|
|
414
|
+
if self.s3_resource is None:
|
|
415
|
+
logger.debug("Cannot check files in s3 zip without s3 resource")
|
|
416
|
+
return bucket_names_and_prefixes
|
|
417
|
+
|
|
418
|
+
s3_client = self.s3_resource.meta.client
|
|
419
|
+
|
|
420
|
+
downloaded = []
|
|
421
|
+
for i, pack in enumerate(bucket_names_and_prefixes):
|
|
422
|
+
bucket_name, prefix = pack
|
|
423
|
+
if ".zip!" in prefix:
|
|
424
|
+
splitted_path = prefix.split(".zip!")
|
|
425
|
+
zip_prefix = f"{splitted_path[0]}.zip"
|
|
426
|
+
rel_path = splitted_path[-1]
|
|
427
|
+
dest_file = os.path.join(product_local_path, rel_path)
|
|
428
|
+
dest_abs_path_dir = os.path.dirname(dest_file)
|
|
429
|
+
if not os.path.isdir(dest_abs_path_dir):
|
|
430
|
+
os.makedirs(dest_abs_path_dir)
|
|
431
|
+
|
|
432
|
+
with open_s3_zipped_object(
|
|
433
|
+
bucket_name, zip_prefix, s3_client, partial=False
|
|
434
|
+
) as zip_file:
|
|
435
|
+
# file size
|
|
436
|
+
file_info = zip_file.getinfo(rel_path)
|
|
437
|
+
progress_callback.reset(total=file_info.file_size)
|
|
438
|
+
with zip_file.open(rel_path) as extracted, open(
|
|
439
|
+
dest_file, "wb"
|
|
440
|
+
) as output_file:
|
|
441
|
+
# Read in 1MB chunks
|
|
442
|
+
for zchunk in iter(lambda: extracted.read(1024 * 1024), b""):
|
|
443
|
+
output_file.write(zchunk)
|
|
444
|
+
progress_callback(len(zchunk))
|
|
445
|
+
|
|
446
|
+
downloaded.append(i)
|
|
447
|
+
|
|
448
|
+
return [
|
|
449
|
+
pack
|
|
450
|
+
for i, pack in enumerate(bucket_names_and_prefixes)
|
|
451
|
+
if i not in downloaded
|
|
452
|
+
]
|
|
453
|
+
|
|
392
454
|
def _download_preparation(
|
|
393
455
|
self,
|
|
394
456
|
product: EOProduct,
|
|
@@ -396,10 +458,12 @@ class AwsDownload(Download):
|
|
|
396
458
|
**kwargs: Unpack[DownloadConf],
|
|
397
459
|
) -> tuple[Optional[str], Optional[str]]:
|
|
398
460
|
"""
|
|
399
|
-
|
|
461
|
+
Preparation for the download:
|
|
462
|
+
|
|
400
463
|
- check if file was already downloaded
|
|
401
464
|
- get file path
|
|
402
465
|
- create directories
|
|
466
|
+
|
|
403
467
|
:param product: product to be downloaded
|
|
404
468
|
:param progress_callback: progress callback to be used
|
|
405
469
|
:param kwargs: additional arguments
|
|
@@ -423,7 +487,8 @@ class AwsDownload(Download):
|
|
|
423
487
|
|
|
424
488
|
def _configure_safe_build(self, build_safe: bool, product: EOProduct):
|
|
425
489
|
"""
|
|
426
|
-
|
|
490
|
+
Updates the product properties with fetch metadata if safe build is enabled
|
|
491
|
+
|
|
427
492
|
:param build_safe: if safe build is enabled
|
|
428
493
|
:param product: product to be updated
|
|
429
494
|
"""
|
|
@@ -513,10 +578,11 @@ class AwsDownload(Download):
|
|
|
513
578
|
auth: Optional[Union[AuthBase, S3SessionKwargs]] = None,
|
|
514
579
|
) -> tuple[dict[str, Any], ResourceCollection]:
|
|
515
580
|
"""
|
|
516
|
-
|
|
517
|
-
|
|
581
|
+
Authenticates with s3 and retrieves the available objects
|
|
582
|
+
|
|
518
583
|
:param bucket_names_and_prefixes: list of bucket names and corresponding path prefixes
|
|
519
584
|
:param auth: authentication information
|
|
585
|
+
:raises AuthenticationError: authentication is not possible
|
|
520
586
|
:return: authenticated objects per bucket, list of available objects
|
|
521
587
|
"""
|
|
522
588
|
if not isinstance(auth, (dict, type(None))):
|
|
@@ -583,14 +649,17 @@ class AwsDownload(Download):
|
|
|
583
649
|
asset_filter: Optional[str],
|
|
584
650
|
ignore_assets: bool,
|
|
585
651
|
product: EOProduct,
|
|
652
|
+
raise_error: bool = True,
|
|
586
653
|
) -> set[Any]:
|
|
587
654
|
"""
|
|
588
|
-
|
|
655
|
+
Retrieve unique product chunks based on authenticated objects and asset filters
|
|
656
|
+
|
|
589
657
|
:param bucket_names_and_prefixes: list of bucket names and corresponding path prefixes
|
|
590
658
|
:param authenticated_objects: available objects per bucket
|
|
591
659
|
:param asset_filter: text for which assets should be filtered
|
|
592
660
|
:param ignore_assets: if product instead of individual assets should be used
|
|
593
661
|
:param product: product that shall be downloaded
|
|
662
|
+
:param raise_error: raise error if there is nothing to download
|
|
594
663
|
:return: set of product chunks that can be downloaded
|
|
595
664
|
"""
|
|
596
665
|
product_chunks: list[Any] = []
|
|
@@ -612,12 +681,12 @@ class AwsDownload(Download):
|
|
|
612
681
|
unique_product_chunks,
|
|
613
682
|
)
|
|
614
683
|
)
|
|
615
|
-
if not unique_product_chunks:
|
|
684
|
+
if not unique_product_chunks and raise_error:
|
|
616
685
|
raise NotAvailableError(
|
|
617
686
|
rf"No file basename matching re.fullmatch(r'{asset_filter}') was found in {product.remote_location}"
|
|
618
687
|
)
|
|
619
688
|
|
|
620
|
-
if not unique_product_chunks:
|
|
689
|
+
if not unique_product_chunks and raise_error:
|
|
621
690
|
raise NoMatchingProductType("No product found to download.")
|
|
622
691
|
|
|
623
692
|
return unique_product_chunks
|
|
@@ -701,6 +770,13 @@ class AwsDownload(Download):
|
|
|
701
770
|
bucket_names_and_prefixes, auth
|
|
702
771
|
)
|
|
703
772
|
|
|
773
|
+
# stream not implemented for prefixes like `foo/bar.zip!file.txt`
|
|
774
|
+
for _, prefix in bucket_names_and_prefixes:
|
|
775
|
+
if prefix and ".zip!" in prefix:
|
|
776
|
+
raise NotImplementedError(
|
|
777
|
+
"Download streaming is not implemented for files in zip on S3"
|
|
778
|
+
)
|
|
779
|
+
|
|
704
780
|
# downloadable files
|
|
705
781
|
unique_product_chunks = self._get_unique_products(
|
|
706
782
|
bucket_names_and_prefixes,
|
|
@@ -935,6 +1011,7 @@ class AwsDownload(Download):
|
|
|
935
1011
|
objects = s3_resource.Bucket(bucket_name).objects
|
|
936
1012
|
list(objects.filter(Prefix=prefix).limit(1))
|
|
937
1013
|
self.s3_session = s3_session
|
|
1014
|
+
self.s3_resource = s3_resource
|
|
938
1015
|
return objects
|
|
939
1016
|
else:
|
|
940
1017
|
return None
|
|
@@ -965,6 +1042,7 @@ class AwsDownload(Download):
|
|
|
965
1042
|
objects = s3_resource.Bucket(bucket_name).objects
|
|
966
1043
|
list(objects.filter(Prefix=prefix).limit(1))
|
|
967
1044
|
self.s3_session = s3_session
|
|
1045
|
+
self.s3_resource = s3_resource
|
|
968
1046
|
return objects
|
|
969
1047
|
else:
|
|
970
1048
|
return None
|
|
@@ -986,6 +1064,7 @@ class AwsDownload(Download):
|
|
|
986
1064
|
objects = s3_resource.Bucket(bucket_name).objects
|
|
987
1065
|
list(objects.filter(Prefix=prefix).limit(1))
|
|
988
1066
|
self.s3_session = s3_session
|
|
1067
|
+
self.s3_resource = s3_resource
|
|
989
1068
|
return objects
|
|
990
1069
|
|
|
991
1070
|
def get_product_bucket_name_and_prefix(
|
eodag/plugins/search/base.py
CHANGED
|
@@ -150,7 +150,7 @@ class Search(PluginTopic):
|
|
|
150
150
|
)
|
|
151
151
|
|
|
152
152
|
def get_product_type_def_params(
|
|
153
|
-
self, product_type: str,
|
|
153
|
+
self, product_type: str, format_variables: Optional[dict[str, Any]] = None
|
|
154
154
|
) -> dict[str, Any]:
|
|
155
155
|
"""Get the provider product type definition parameters and specific settings
|
|
156
156
|
|
|
@@ -171,7 +171,8 @@ class Search(PluginTopic):
|
|
|
171
171
|
return {
|
|
172
172
|
k: v
|
|
173
173
|
for k, v in format_dict_items(
|
|
174
|
-
self.config.products[GENERIC_PRODUCT_TYPE],
|
|
174
|
+
self.config.products[GENERIC_PRODUCT_TYPE],
|
|
175
|
+
**(format_variables or {}),
|
|
175
176
|
).items()
|
|
176
177
|
if v
|
|
177
178
|
}
|