eodag 3.10.1__py3-none-any.whl → 4.0.0a2__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- eodag/__init__.py +6 -1
- eodag/api/collection.py +353 -0
- eodag/api/core.py +606 -641
- eodag/api/product/__init__.py +3 -3
- eodag/api/product/_product.py +74 -56
- eodag/api/product/drivers/__init__.py +4 -46
- eodag/api/product/drivers/base.py +0 -28
- eodag/api/product/metadata_mapping.py +178 -216
- eodag/api/search_result.py +156 -15
- eodag/cli.py +83 -403
- eodag/config.py +81 -51
- eodag/plugins/apis/base.py +2 -2
- eodag/plugins/apis/ecmwf.py +36 -25
- eodag/plugins/apis/usgs.py +55 -40
- eodag/plugins/authentication/base.py +1 -3
- eodag/plugins/crunch/filter_date.py +3 -3
- eodag/plugins/crunch/filter_latest_intersect.py +2 -2
- eodag/plugins/crunch/filter_latest_tpl_name.py +1 -1
- eodag/plugins/download/aws.py +46 -42
- eodag/plugins/download/base.py +13 -14
- eodag/plugins/download/http.py +65 -65
- eodag/plugins/manager.py +28 -29
- eodag/plugins/search/__init__.py +6 -4
- eodag/plugins/search/base.py +131 -80
- eodag/plugins/search/build_search_result.py +245 -173
- eodag/plugins/search/cop_marine.py +87 -56
- eodag/plugins/search/csw.py +47 -37
- eodag/plugins/search/qssearch.py +653 -429
- eodag/plugins/search/stac_list_assets.py +1 -1
- eodag/plugins/search/static_stac_search.py +43 -44
- eodag/resources/{product_types.yml → collections.yml} +2594 -2453
- eodag/resources/ext_collections.json +1 -1
- eodag/resources/ext_product_types.json +1 -1
- eodag/resources/providers.yml +2706 -2733
- eodag/resources/stac_provider.yml +50 -92
- eodag/resources/user_conf_template.yml +9 -0
- eodag/types/__init__.py +2 -0
- eodag/types/queryables.py +70 -91
- eodag/types/search_args.py +1 -1
- eodag/utils/__init__.py +97 -21
- eodag/utils/dates.py +0 -12
- eodag/utils/exceptions.py +6 -6
- eodag/utils/free_text_search.py +3 -3
- eodag/utils/repr.py +2 -0
- {eodag-3.10.1.dist-info → eodag-4.0.0a2.dist-info}/METADATA +13 -99
- eodag-4.0.0a2.dist-info/RECORD +93 -0
- {eodag-3.10.1.dist-info → eodag-4.0.0a2.dist-info}/entry_points.txt +0 -4
- eodag/plugins/authentication/oauth.py +0 -60
- eodag/plugins/download/creodias_s3.py +0 -71
- eodag/plugins/download/s3rest.py +0 -351
- eodag/plugins/search/data_request_search.py +0 -565
- eodag/resources/stac.yml +0 -294
- eodag/resources/stac_api.yml +0 -2105
- eodag/rest/__init__.py +0 -24
- eodag/rest/cache.py +0 -70
- eodag/rest/config.py +0 -67
- eodag/rest/constants.py +0 -26
- eodag/rest/core.py +0 -764
- eodag/rest/errors.py +0 -210
- eodag/rest/server.py +0 -604
- eodag/rest/server.wsgi +0 -6
- eodag/rest/stac.py +0 -1032
- eodag/rest/templates/README +0 -1
- eodag/rest/types/__init__.py +0 -18
- eodag/rest/types/collections_search.py +0 -44
- eodag/rest/types/eodag_search.py +0 -386
- eodag/rest/types/queryables.py +0 -174
- eodag/rest/types/stac_search.py +0 -272
- eodag/rest/utils/__init__.py +0 -207
- eodag/rest/utils/cql_evaluate.py +0 -119
- eodag/rest/utils/rfc3339.py +0 -64
- eodag-3.10.1.dist-info/RECORD +0 -116
- {eodag-3.10.1.dist-info → eodag-4.0.0a2.dist-info}/WHEEL +0 -0
- {eodag-3.10.1.dist-info → eodag-4.0.0a2.dist-info}/licenses/LICENSE +0 -0
- {eodag-3.10.1.dist-info → eodag-4.0.0a2.dist-info}/top_level.txt +0 -0
eodag/plugins/apis/usgs.py
CHANGED
|
@@ -31,10 +31,10 @@ from usgs import USGSAuthExpiredError, USGSError, api
|
|
|
31
31
|
|
|
32
32
|
from eodag.api.product import EOProduct
|
|
33
33
|
from eodag.api.product.metadata_mapping import (
|
|
34
|
-
DEFAULT_METADATA_MAPPING,
|
|
35
34
|
mtd_cfg_as_conversion_and_querypath,
|
|
36
35
|
properties_from_json,
|
|
37
36
|
)
|
|
37
|
+
from eodag.api.search_result import SearchResult
|
|
38
38
|
from eodag.plugins.apis.base import Api
|
|
39
39
|
from eodag.plugins.search import PreparedSearch
|
|
40
40
|
from eodag.utils import (
|
|
@@ -42,7 +42,7 @@ from eodag.utils import (
|
|
|
42
42
|
DEFAULT_DOWNLOAD_WAIT,
|
|
43
43
|
DEFAULT_ITEMS_PER_PAGE,
|
|
44
44
|
DEFAULT_PAGE,
|
|
45
|
-
|
|
45
|
+
GENERIC_COLLECTION,
|
|
46
46
|
USER_AGENT,
|
|
47
47
|
ProgressCallback,
|
|
48
48
|
format_dict_items,
|
|
@@ -50,7 +50,7 @@ from eodag.utils import (
|
|
|
50
50
|
)
|
|
51
51
|
from eodag.utils.exceptions import (
|
|
52
52
|
AuthenticationError,
|
|
53
|
-
|
|
53
|
+
NoMatchingCollection,
|
|
54
54
|
NotAvailableError,
|
|
55
55
|
RequestError,
|
|
56
56
|
ValidationError,
|
|
@@ -60,9 +60,7 @@ if TYPE_CHECKING:
|
|
|
60
60
|
from mypy_boto3_s3 import S3ServiceResource
|
|
61
61
|
from requests.auth import AuthBase
|
|
62
62
|
|
|
63
|
-
from eodag.api.search_result import SearchResult
|
|
64
63
|
from eodag.config import PluginConfig
|
|
65
|
-
from eodag.types import S3SessionKwargs
|
|
66
64
|
from eodag.types.download_args import DownloadConf
|
|
67
65
|
from eodag.utils import DownloadedCallback, Unpack
|
|
68
66
|
|
|
@@ -101,7 +99,7 @@ class UsgsApi(Api):
|
|
|
101
99
|
# Same method as in base.py, Search.__init__()
|
|
102
100
|
# Prepare the metadata mapping
|
|
103
101
|
# Do a shallow copy, the structure is flat enough for this to be sufficient
|
|
104
|
-
metas: dict[str, Any] =
|
|
102
|
+
metas: dict[str, Any] = {}
|
|
105
103
|
# Update the defaults with the mapping value. This will add any new key
|
|
106
104
|
# added by the provider mapping that is not in the default metadata.
|
|
107
105
|
metas.update(self.config.metadata_mapping)
|
|
@@ -140,31 +138,38 @@ class UsgsApi(Api):
|
|
|
140
138
|
self,
|
|
141
139
|
prep: PreparedSearch = PreparedSearch(),
|
|
142
140
|
**kwargs: Any,
|
|
143
|
-
) ->
|
|
141
|
+
) -> SearchResult:
|
|
144
142
|
"""Search for data on USGS catalogues"""
|
|
145
|
-
|
|
143
|
+
token = (
|
|
144
|
+
int(prep.next_page_token)
|
|
145
|
+
if prep.next_page_token is not None
|
|
146
|
+
else DEFAULT_PAGE
|
|
147
|
+
)
|
|
146
148
|
items_per_page = (
|
|
147
149
|
prep.items_per_page
|
|
148
|
-
|
|
149
|
-
|
|
150
|
+
or kwargs.pop("max_results", None)
|
|
151
|
+
or DEFAULT_ITEMS_PER_PAGE
|
|
150
152
|
)
|
|
151
|
-
|
|
152
|
-
|
|
153
|
-
|
|
154
|
-
|
|
153
|
+
search_params = {"items_per_page": items_per_page} | kwargs
|
|
154
|
+
collection = kwargs.get("collection")
|
|
155
|
+
if collection is None:
|
|
156
|
+
raise NoMatchingCollection(
|
|
157
|
+
"Cannot search on USGS without collection specified"
|
|
155
158
|
)
|
|
156
159
|
if kwargs.get("sort_by"):
|
|
157
160
|
raise ValidationError("USGS does not support sorting feature")
|
|
158
161
|
|
|
159
162
|
self.authenticate()
|
|
160
163
|
|
|
161
|
-
|
|
162
|
-
|
|
163
|
-
self.config.products[
|
|
164
|
+
collection_def_params = self.config.products.get( # type: ignore
|
|
165
|
+
collection,
|
|
166
|
+
self.config.products[GENERIC_COLLECTION], # type: ignore
|
|
164
167
|
)
|
|
165
|
-
|
|
166
|
-
|
|
167
|
-
|
|
168
|
+
usgs_collection = format_dict_items(collection_def_params, **kwargs)[
|
|
169
|
+
"_collection"
|
|
170
|
+
]
|
|
171
|
+
start_date = kwargs.pop("start_datetime", None)
|
|
172
|
+
end_date = kwargs.pop("end_datetime", None)
|
|
168
173
|
geom = kwargs.pop("geometry", None)
|
|
169
174
|
footprint: dict[str, str] = {}
|
|
170
175
|
if hasattr(geom, "bounds"):
|
|
@@ -196,19 +201,21 @@ class UsgsApi(Api):
|
|
|
196
201
|
ll=lower_left,
|
|
197
202
|
ur=upper_right,
|
|
198
203
|
max_results=items_per_page,
|
|
199
|
-
starting_number=
|
|
204
|
+
starting_number=token,
|
|
200
205
|
)
|
|
201
206
|
|
|
202
207
|
# search by id
|
|
203
208
|
if searched_id := kwargs.get("id"):
|
|
204
|
-
dataset_filters = api.dataset_filters(
|
|
209
|
+
dataset_filters = api.dataset_filters(usgs_collection)
|
|
205
210
|
# ip pattern set as parameter queryable (first element of param conf list)
|
|
206
211
|
id_pattern = self.config.metadata_mapping["id"][0]
|
|
207
212
|
# loop on matching dataset_filters until one returns expected results
|
|
208
213
|
for dataset_filter in dataset_filters["data"]:
|
|
209
214
|
if id_pattern in dataset_filter["searchSql"]:
|
|
210
215
|
logger.debug(
|
|
211
|
-
|
|
216
|
+
"Try using %s dataset filter to search by id on %s",
|
|
217
|
+
dataset_filter["searchSql"],
|
|
218
|
+
usgs_collection,
|
|
212
219
|
)
|
|
213
220
|
full_api_search_kwargs = {
|
|
214
221
|
"where": {
|
|
@@ -218,19 +225,19 @@ class UsgsApi(Api):
|
|
|
218
225
|
**api_search_kwargs,
|
|
219
226
|
}
|
|
220
227
|
logger.info(
|
|
221
|
-
f"Sending search request for {
|
|
228
|
+
f"Sending search request for {usgs_collection} with {full_api_search_kwargs}"
|
|
222
229
|
)
|
|
223
230
|
results = api.scene_search(
|
|
224
|
-
|
|
231
|
+
usgs_collection, **full_api_search_kwargs
|
|
225
232
|
)
|
|
226
233
|
if len(results["data"]["results"]) == 1:
|
|
227
234
|
# search by id using this dataset_filter succeeded
|
|
228
235
|
break
|
|
229
236
|
else:
|
|
230
237
|
logger.info(
|
|
231
|
-
f"Sending search request for {
|
|
238
|
+
f"Sending search request for {usgs_collection} with {api_search_kwargs}"
|
|
232
239
|
)
|
|
233
|
-
results = api.scene_search(
|
|
240
|
+
results = api.scene_search(usgs_collection, **api_search_kwargs)
|
|
234
241
|
|
|
235
242
|
# update results with storage info from download_options()
|
|
236
243
|
results_by_entity_id = {
|
|
@@ -240,7 +247,7 @@ class UsgsApi(Api):
|
|
|
240
247
|
f"Adapting {len(results_by_entity_id)} plugin results to eodag product representation"
|
|
241
248
|
)
|
|
242
249
|
download_options = api.download_options(
|
|
243
|
-
|
|
250
|
+
usgs_collection, list(results_by_entity_id.keys())
|
|
244
251
|
)
|
|
245
252
|
if download_options.get("data") is not None:
|
|
246
253
|
for download_option in download_options["data"]:
|
|
@@ -262,7 +269,7 @@ class UsgsApi(Api):
|
|
|
262
269
|
results["data"]["results"] = list(results_by_entity_id.values())
|
|
263
270
|
|
|
264
271
|
for result in results["data"]["results"]:
|
|
265
|
-
result["
|
|
272
|
+
result["collection"] = usgs_collection
|
|
266
273
|
|
|
267
274
|
product_properties = properties_from_json(
|
|
268
275
|
result, self.config.metadata_mapping
|
|
@@ -270,7 +277,7 @@ class UsgsApi(Api):
|
|
|
270
277
|
|
|
271
278
|
final.append(
|
|
272
279
|
EOProduct(
|
|
273
|
-
|
|
280
|
+
collection=collection,
|
|
274
281
|
provider=self.provider,
|
|
275
282
|
properties=product_properties,
|
|
276
283
|
geometry=footprint,
|
|
@@ -278,7 +285,7 @@ class UsgsApi(Api):
|
|
|
278
285
|
)
|
|
279
286
|
except USGSError as e:
|
|
280
287
|
logger.warning(
|
|
281
|
-
f"
|
|
288
|
+
f"Collection {usgs_collection} may not exist on USGS EE catalog"
|
|
282
289
|
)
|
|
283
290
|
api.logout()
|
|
284
291
|
raise RequestError.from_error(e) from e
|
|
@@ -292,12 +299,18 @@ class UsgsApi(Api):
|
|
|
292
299
|
else:
|
|
293
300
|
total_results = 0
|
|
294
301
|
|
|
295
|
-
|
|
302
|
+
formated_result = SearchResult(
|
|
303
|
+
final,
|
|
304
|
+
total_results,
|
|
305
|
+
search_params=search_params,
|
|
306
|
+
next_page_token=results["data"]["nextRecord"],
|
|
307
|
+
)
|
|
308
|
+
return formated_result
|
|
296
309
|
|
|
297
310
|
def download(
|
|
298
311
|
self,
|
|
299
312
|
product: EOProduct,
|
|
300
|
-
auth: Optional[Union[AuthBase,
|
|
313
|
+
auth: Optional[Union[AuthBase, S3ServiceResource]] = None,
|
|
301
314
|
progress_callback: Optional[ProgressCallback] = None,
|
|
302
315
|
wait: float = DEFAULT_DOWNLOAD_WAIT,
|
|
303
316
|
timeout: float = DEFAULT_DOWNLOAD_TIMEOUT,
|
|
@@ -314,7 +327,7 @@ class UsgsApi(Api):
|
|
|
314
327
|
output_extension = cast(
|
|
315
328
|
str,
|
|
316
329
|
self.config.products.get( # type: ignore
|
|
317
|
-
product.
|
|
330
|
+
product.collection, self.config.products[GENERIC_COLLECTION] # type: ignore
|
|
318
331
|
).get("output_extension", ".tar.gz"),
|
|
319
332
|
)
|
|
320
333
|
kwargs["output_extension"] = kwargs.get("output_extension", output_extension)
|
|
@@ -335,11 +348,13 @@ class UsgsApi(Api):
|
|
|
335
348
|
raise NotAvailableError(
|
|
336
349
|
f"No USGS products found for {product.properties['id']}"
|
|
337
350
|
)
|
|
338
|
-
|
|
351
|
+
usgs_dataset = self.config.products.get(product.collection, {}).get(
|
|
352
|
+
"_collection", GENERIC_COLLECTION
|
|
353
|
+
)
|
|
339
354
|
download_request_results = api.download_request(
|
|
340
|
-
|
|
341
|
-
product.properties["entityId"],
|
|
342
|
-
product.properties["productId"],
|
|
355
|
+
usgs_dataset,
|
|
356
|
+
product.properties["usgs:entityId"],
|
|
357
|
+
product.properties["usgs:productId"],
|
|
343
358
|
)
|
|
344
359
|
|
|
345
360
|
req_urls: list[str] = []
|
|
@@ -422,7 +437,7 @@ class UsgsApi(Api):
|
|
|
422
437
|
download_request(product, fs_path, progress_callback, **kwargs)
|
|
423
438
|
|
|
424
439
|
with open(record_filename, "w") as fh:
|
|
425
|
-
fh.write(product.properties["
|
|
440
|
+
fh.write(product.properties["eodag:download_link"])
|
|
426
441
|
logger.debug(f"Download recorded in {record_filename}")
|
|
427
442
|
|
|
428
443
|
api.logout()
|
|
@@ -466,7 +481,7 @@ class UsgsApi(Api):
|
|
|
466
481
|
def download_all(
|
|
467
482
|
self,
|
|
468
483
|
products: SearchResult,
|
|
469
|
-
auth: Optional[Union[AuthBase,
|
|
484
|
+
auth: Optional[Union[AuthBase, S3ServiceResource]] = None,
|
|
470
485
|
downloaded_callback: Optional[DownloadedCallback] = None,
|
|
471
486
|
progress_callback: Optional[ProgressCallback] = None,
|
|
472
487
|
wait: float = DEFAULT_DOWNLOAD_WAIT,
|
|
@@ -27,8 +27,6 @@ if TYPE_CHECKING:
|
|
|
27
27
|
from mypy_boto3_s3 import S3ServiceResource
|
|
28
28
|
from requests.auth import AuthBase
|
|
29
29
|
|
|
30
|
-
from eodag.types import S3SessionKwargs
|
|
31
|
-
|
|
32
30
|
|
|
33
31
|
class Authentication(PluginTopic):
|
|
34
32
|
"""Plugins authentication Base plugin
|
|
@@ -42,7 +40,7 @@ class Authentication(PluginTopic):
|
|
|
42
40
|
configuration that needs authentication and helps identifying it
|
|
43
41
|
"""
|
|
44
42
|
|
|
45
|
-
def authenticate(self) -> Union[AuthBase,
|
|
43
|
+
def authenticate(self) -> Union[AuthBase, S3ServiceResource]:
|
|
46
44
|
"""Authenticate"""
|
|
47
45
|
raise NotImplementedError
|
|
48
46
|
|
|
@@ -49,7 +49,7 @@ class FilterDate(Crunch):
|
|
|
49
49
|
@staticmethod
|
|
50
50
|
def sort_product_by_start_date(product: EOProduct) -> dt:
|
|
51
51
|
"""Get product start date"""
|
|
52
|
-
start_date = product.properties.get("
|
|
52
|
+
start_date = product.properties.get("start_datetime")
|
|
53
53
|
if not start_date:
|
|
54
54
|
# Retrieve year, month, day, hour, minute, second of EPOCH start
|
|
55
55
|
epoch = time.gmtime(0)[:-3]
|
|
@@ -93,7 +93,7 @@ class FilterDate(Crunch):
|
|
|
93
93
|
for product in products:
|
|
94
94
|
|
|
95
95
|
# product start date
|
|
96
|
-
product_start_str = product.properties.get("
|
|
96
|
+
product_start_str = product.properties.get("start_datetime")
|
|
97
97
|
if product_start_str:
|
|
98
98
|
product_start = dateutil.parser.parse(product_start_str)
|
|
99
99
|
if not product_start.tzinfo:
|
|
@@ -102,7 +102,7 @@ class FilterDate(Crunch):
|
|
|
102
102
|
product_start = None
|
|
103
103
|
|
|
104
104
|
# product end date
|
|
105
|
-
product_end_str = product.properties.get("
|
|
105
|
+
product_end_str = product.properties.get("end_datetime")
|
|
106
106
|
if product_end_str:
|
|
107
107
|
product_end = dateutil.parser.parse(product_end_str)
|
|
108
108
|
if not product_end.tzinfo:
|
|
@@ -46,7 +46,7 @@ class FilterLatestIntersect(Crunch):
|
|
|
46
46
|
@staticmethod
|
|
47
47
|
def sort_product_by_start_date(product: EOProduct) -> dt:
|
|
48
48
|
"""Get product start date"""
|
|
49
|
-
start_date = product.properties.get("
|
|
49
|
+
start_date = product.properties.get("start_datetime")
|
|
50
50
|
if not start_date:
|
|
51
51
|
# Retrieve year, month, day, hour, minute, second of EPOCH start
|
|
52
52
|
epoch = time.gmtime(0)[:-3]
|
|
@@ -67,7 +67,7 @@ class FilterLatestIntersect(Crunch):
|
|
|
67
67
|
logger.debug("Start filtering for latest products")
|
|
68
68
|
if not products:
|
|
69
69
|
return []
|
|
70
|
-
# Warning: May crash if
|
|
70
|
+
# Warning: May crash if start_datetime is not in the appropriate format
|
|
71
71
|
products.sort(key=self.sort_product_by_start_date, reverse=True)
|
|
72
72
|
filtered: list[EOProduct] = []
|
|
73
73
|
search_extent: BaseGeometry
|
|
@@ -75,7 +75,7 @@ class FilterLatestByName(Crunch):
|
|
|
75
75
|
logger.debug(
|
|
76
76
|
"Latest product found for tileid=%s: date=%s",
|
|
77
77
|
tileid,
|
|
78
|
-
product.properties["
|
|
78
|
+
product.properties["start_datetime"],
|
|
79
79
|
)
|
|
80
80
|
filtered.append(product)
|
|
81
81
|
processed.append(tileid)
|
eodag/plugins/download/aws.py
CHANGED
|
@@ -44,6 +44,7 @@ from eodag.utils import (
|
|
|
44
44
|
ProgressCallback,
|
|
45
45
|
StreamResponse,
|
|
46
46
|
flatten_top_directories,
|
|
47
|
+
format_string,
|
|
47
48
|
get_bucket_name_and_prefix,
|
|
48
49
|
path_to_uri,
|
|
49
50
|
rename_subfolder,
|
|
@@ -53,7 +54,7 @@ from eodag.utils.exceptions import (
|
|
|
53
54
|
AuthenticationError,
|
|
54
55
|
DownloadError,
|
|
55
56
|
MisconfiguredError,
|
|
56
|
-
|
|
57
|
+
NoMatchingCollection,
|
|
57
58
|
NotAvailableError,
|
|
58
59
|
TimeOutError,
|
|
59
60
|
)
|
|
@@ -66,7 +67,6 @@ if TYPE_CHECKING:
|
|
|
66
67
|
from eodag.api.product import EOProduct
|
|
67
68
|
from eodag.api.search_result import SearchResult
|
|
68
69
|
from eodag.config import PluginConfig
|
|
69
|
-
from eodag.types import S3SessionKwargs
|
|
70
70
|
from eodag.types.download_args import DownloadConf
|
|
71
71
|
from eodag.utils import DownloadedCallback, Unpack
|
|
72
72
|
|
|
@@ -202,17 +202,17 @@ class AwsDownload(Download):
|
|
|
202
202
|
* :attr:`~eodag.config.PluginConfig.flatten_top_dirs` (``bool``): if the directory structure
|
|
203
203
|
should be flattened; default: ``True``
|
|
204
204
|
* :attr:`~eodag.config.PluginConfig.ignore_assets` (``bool``): ignore assets and download
|
|
205
|
-
using ``
|
|
205
|
+
using ``eodag:download_link``; default: ``False``
|
|
206
206
|
* :attr:`~eodag.config.PluginConfig.ssl_verify` (``bool``): if the ssl certificates should
|
|
207
207
|
be verified in requests; default: ``True``
|
|
208
208
|
* :attr:`~eodag.config.PluginConfig.bucket_path_level` (``int``): at which level of the
|
|
209
209
|
path part of the url the bucket can be found; If no bucket_path_level is given, the bucket
|
|
210
210
|
is taken from the first element of the netloc part.
|
|
211
|
-
* :attr:`~eodag.config.PluginConfig.products` (``dict[str, dict[str, Any]``):
|
|
212
|
-
specific config; the keys are the
|
|
211
|
+
* :attr:`~eodag.config.PluginConfig.products` (``dict[str, dict[str, Any]``): collection
|
|
212
|
+
specific config; the keys are the collections, the values are dictionaries which can contain the keys:
|
|
213
213
|
|
|
214
|
-
* **default_bucket** (``str``): bucket where the
|
|
215
|
-
* **complementary_url_key** (``str``): keys to
|
|
214
|
+
* **default_bucket** (``str``): bucket where the collection can be found
|
|
215
|
+
* **complementary_url_key** (``str``): properties keys pointing to additional urls of content to download
|
|
216
216
|
* **build_safe** (``bool``): if a SAFE (Standard Archive Format for Europe) product should
|
|
217
217
|
be created; used for Sentinel products; default: False
|
|
218
218
|
* **fetch_metadata** (``dict[str, Any]``): config for metadata to be fetched for the SAFE product
|
|
@@ -225,7 +225,7 @@ class AwsDownload(Download):
|
|
|
225
225
|
def download(
|
|
226
226
|
self,
|
|
227
227
|
product: EOProduct,
|
|
228
|
-
auth: Optional[Union[AuthBase,
|
|
228
|
+
auth: Optional[Union[AuthBase, S3ServiceResource]] = None,
|
|
229
229
|
progress_callback: Optional[ProgressCallback] = None,
|
|
230
230
|
wait: float = DEFAULT_DOWNLOAD_WAIT,
|
|
231
231
|
timeout: float = DEFAULT_DOWNLOAD_TIMEOUT,
|
|
@@ -234,7 +234,7 @@ class AwsDownload(Download):
|
|
|
234
234
|
"""Download method for AWS S3 API.
|
|
235
235
|
|
|
236
236
|
The product can be downloaded as it is, or as SAFE-formatted product.
|
|
237
|
-
SAFE-build is configured for a given provider and
|
|
237
|
+
SAFE-build is configured for a given provider and collection.
|
|
238
238
|
If the product title is configured to be updated during download and
|
|
239
239
|
SAFE-formatted, its destination path will be:
|
|
240
240
|
`{output_dir}/{title}`
|
|
@@ -266,9 +266,7 @@ class AwsDownload(Download):
|
|
|
266
266
|
if not record_filename or not product_local_path:
|
|
267
267
|
return product_local_path
|
|
268
268
|
|
|
269
|
-
product_conf = getattr(self.config, "products", {}).get(
|
|
270
|
-
product.product_type, {}
|
|
271
|
-
)
|
|
269
|
+
product_conf = getattr(self.config, "products", {}).get(product.collection, {})
|
|
272
270
|
|
|
273
271
|
# do not try to build SAFE if asset filter is used
|
|
274
272
|
asset_filter = kwargs.get("asset")
|
|
@@ -368,7 +366,7 @@ class AwsDownload(Download):
|
|
|
368
366
|
logger.warning("Unexpected error: %s" % e)
|
|
369
367
|
|
|
370
368
|
# finalize safe product
|
|
371
|
-
if build_safe and product.
|
|
369
|
+
if build_safe and product.collection and "S2_MSI" in product.collection:
|
|
372
370
|
self.finalize_s2_safe_product(product_local_path)
|
|
373
371
|
# flatten directory structure
|
|
374
372
|
elif flatten_top_dirs:
|
|
@@ -479,17 +477,15 @@ class AwsDownload(Download):
|
|
|
479
477
|
:param build_safe: if safe build is enabled
|
|
480
478
|
:param product: product to be updated
|
|
481
479
|
"""
|
|
482
|
-
product_conf = getattr(self.config, "products", {}).get(
|
|
483
|
-
product.product_type, {}
|
|
484
|
-
)
|
|
480
|
+
product_conf = getattr(self.config, "products", {}).get(product.collection, {})
|
|
485
481
|
ssl_verify = getattr(self.config, "ssl_verify", True)
|
|
486
482
|
timeout = getattr(self.config, "timeout", HTTP_REQ_TIMEOUT)
|
|
487
483
|
|
|
488
484
|
if build_safe and "fetch_metadata" in product_conf.keys():
|
|
489
485
|
fetch_format = product_conf["fetch_metadata"]["fetch_format"]
|
|
490
486
|
update_metadata = product_conf["fetch_metadata"]["update_metadata"]
|
|
491
|
-
fetch_url =
|
|
492
|
-
**product.properties
|
|
487
|
+
fetch_url = format_string(
|
|
488
|
+
None, product_conf["fetch_metadata"]["fetch_url"], **product.properties
|
|
493
489
|
)
|
|
494
490
|
logger.info("Fetching extra metadata from %s" % fetch_url)
|
|
495
491
|
try:
|
|
@@ -619,14 +615,14 @@ class AwsDownload(Download):
|
|
|
619
615
|
)
|
|
620
616
|
|
|
621
617
|
if not unique_product_chunks and raise_error:
|
|
622
|
-
raise
|
|
618
|
+
raise NoMatchingCollection("No product found to download.")
|
|
623
619
|
|
|
624
620
|
return unique_product_chunks
|
|
625
621
|
|
|
626
622
|
def _stream_download_dict(
|
|
627
623
|
self,
|
|
628
624
|
product: EOProduct,
|
|
629
|
-
auth: Optional[Union[AuthBase,
|
|
625
|
+
auth: Optional[Union[AuthBase, S3ServiceResource]] = None,
|
|
630
626
|
byte_range: tuple[Optional[int], Optional[int]] = (None, None),
|
|
631
627
|
compress: Literal["zip", "raw", "auto"] = "auto",
|
|
632
628
|
wait: float = DEFAULT_DOWNLOAD_WAIT,
|
|
@@ -664,7 +660,7 @@ class AwsDownload(Download):
|
|
|
664
660
|
|
|
665
661
|
#### SAFE Archive Support:
|
|
666
662
|
|
|
667
|
-
If the
|
|
663
|
+
If the collection supports SAFE structure and no `asset_regex` is specified (i.e., full product download),
|
|
668
664
|
the method attempts to reconstruct a valid SAFE archive layout in the streamed output.
|
|
669
665
|
|
|
670
666
|
:param product: The EO product to download.
|
|
@@ -680,9 +676,7 @@ class AwsDownload(Download):
|
|
|
680
676
|
"""
|
|
681
677
|
asset_regex = kwargs.get("asset")
|
|
682
678
|
|
|
683
|
-
product_conf = getattr(self.config, "products", {}).get(
|
|
684
|
-
product.product_type, {}
|
|
685
|
-
)
|
|
679
|
+
product_conf = getattr(self.config, "products", {}).get(product.collection, {})
|
|
686
680
|
|
|
687
681
|
build_safe = (
|
|
688
682
|
False if asset_regex is not None else product_conf.get("build_safe", False)
|
|
@@ -716,7 +710,7 @@ class AwsDownload(Download):
|
|
|
716
710
|
ignore_assets,
|
|
717
711
|
product,
|
|
718
712
|
)
|
|
719
|
-
if auth and isinstance(auth, boto3.
|
|
713
|
+
if auth and isinstance(auth, boto3.resources.base.ServiceResource):
|
|
720
714
|
s3_resource = auth
|
|
721
715
|
else:
|
|
722
716
|
s3_resource = boto3.resource(
|
|
@@ -724,9 +718,7 @@ class AwsDownload(Download):
|
|
|
724
718
|
endpoint_url=getattr(self.config, "s3_endpoint", None),
|
|
725
719
|
)
|
|
726
720
|
|
|
727
|
-
product_conf = getattr(self.config, "products", {}).get(
|
|
728
|
-
product.product_type, {}
|
|
729
|
-
)
|
|
721
|
+
product_conf = getattr(self.config, "products", {}).get(product.collection, {})
|
|
730
722
|
flatten_top_dirs = product_conf.get(
|
|
731
723
|
"flatten_top_dirs", getattr(self.config, "flatten_top_dirs", True)
|
|
732
724
|
)
|
|
@@ -814,7 +806,7 @@ class AwsDownload(Download):
|
|
|
814
806
|
if bucket is None:
|
|
815
807
|
bucket = (
|
|
816
808
|
getattr(self.config, "products", {})
|
|
817
|
-
.get(product.
|
|
809
|
+
.get(product.collection, {})
|
|
818
810
|
.get("default_bucket", "")
|
|
819
811
|
)
|
|
820
812
|
|
|
@@ -925,7 +917,7 @@ class AwsDownload(Download):
|
|
|
925
917
|
s2_processing_level: str = ""
|
|
926
918
|
s1_title_suffix: Optional[str] = None
|
|
927
919
|
# S2 common
|
|
928
|
-
if product.
|
|
920
|
+
if product.collection and "S2_MSI" in product.collection:
|
|
929
921
|
title_search: Optional[re.Match[str]] = re.search(
|
|
930
922
|
r"^\w+_\w+_(\w+)_(\w+)_(\w+)_(\w+)_(\w+)$",
|
|
931
923
|
product.properties["title"],
|
|
@@ -937,9 +929,9 @@ class AwsDownload(Download):
|
|
|
937
929
|
product.properties.get("originalSceneID", ""),
|
|
938
930
|
)
|
|
939
931
|
ds_dir = ds_dir_search.group(1) if ds_dir_search else 0
|
|
940
|
-
s2_processing_level = product.
|
|
932
|
+
s2_processing_level = product.collection.split("_")[-1]
|
|
941
933
|
# S1 common
|
|
942
|
-
elif product.
|
|
934
|
+
elif product.collection == "S1_SAR_GRD":
|
|
943
935
|
s1_title_suffix_search = re.search(
|
|
944
936
|
r"^.+_([A-Z0-9_]+_[A-Z0-9_]+_[A-Z0-9_]+_[A-Z0-9_]+)_\w+$",
|
|
945
937
|
product.properties["title"],
|
|
@@ -950,6 +942,12 @@ class AwsDownload(Download):
|
|
|
950
942
|
else None
|
|
951
943
|
)
|
|
952
944
|
|
|
945
|
+
# S1 polarization mode
|
|
946
|
+
if product_title := product.properties.get("title"):
|
|
947
|
+
polarization_mode = re.sub(r".{14}([A-Z]{2}).*", r"\1", product_title)
|
|
948
|
+
else:
|
|
949
|
+
polarization_mode = None
|
|
950
|
+
|
|
953
951
|
# S2 L2A Tile files -----------------------------------------------
|
|
954
952
|
if matched := S2L2A_TILE_IMG_REGEX.match(chunk.key):
|
|
955
953
|
found_dict = matched.groupdict()
|
|
@@ -1057,37 +1055,43 @@ class AwsDownload(Download):
|
|
|
1057
1055
|
found_dict = matched.groupdict()
|
|
1058
1056
|
product_path = "%s" % found_dict["file"]
|
|
1059
1057
|
# S1 --------------------------------------------------------------
|
|
1060
|
-
elif
|
|
1058
|
+
elif (
|
|
1059
|
+
matched := S1_CALIB_REGEX.match(chunk.key)
|
|
1060
|
+
) and polarization_mode is not None:
|
|
1061
1061
|
found_dict = matched.groupdict()
|
|
1062
1062
|
product_path = "annotation/calibration/%s-%s-%s-grd-%s-%s-%03d.xml" % (
|
|
1063
1063
|
found_dict["file_prefix"],
|
|
1064
|
-
product.properties["
|
|
1064
|
+
product.properties["platform"].lower(),
|
|
1065
1065
|
found_dict["file_beam"],
|
|
1066
1066
|
found_dict["file_pol"],
|
|
1067
1067
|
s1_title_suffix,
|
|
1068
|
-
S1_IMG_NB_PER_POLAR.get(
|
|
1068
|
+
S1_IMG_NB_PER_POLAR.get(polarization_mode, {}).get(
|
|
1069
1069
|
found_dict["file_pol"].upper(), 1
|
|
1070
1070
|
),
|
|
1071
1071
|
)
|
|
1072
|
-
elif
|
|
1072
|
+
elif (
|
|
1073
|
+
matched := S1_ANNOT_REGEX.match(chunk.key)
|
|
1074
|
+
) and polarization_mode is not None:
|
|
1073
1075
|
found_dict = matched.groupdict()
|
|
1074
1076
|
product_path = "annotation/%s-%s-grd-%s-%s-%03d.xml" % (
|
|
1075
|
-
product.properties["
|
|
1077
|
+
product.properties["platform"].lower(),
|
|
1076
1078
|
found_dict["file_beam"],
|
|
1077
1079
|
found_dict["file_pol"],
|
|
1078
1080
|
s1_title_suffix,
|
|
1079
|
-
S1_IMG_NB_PER_POLAR.get(
|
|
1081
|
+
S1_IMG_NB_PER_POLAR.get(polarization_mode, {}).get(
|
|
1080
1082
|
found_dict["file_pol"].upper(), 1
|
|
1081
1083
|
),
|
|
1082
1084
|
)
|
|
1083
|
-
elif
|
|
1085
|
+
elif (
|
|
1086
|
+
matched := S1_MEAS_REGEX.match(chunk.key)
|
|
1087
|
+
) and polarization_mode is not None:
|
|
1084
1088
|
found_dict = matched.groupdict()
|
|
1085
1089
|
product_path = "measurement/%s-%s-grd-%s-%s-%03d.%s" % (
|
|
1086
|
-
product.properties["
|
|
1090
|
+
product.properties["platform"].lower(),
|
|
1087
1091
|
found_dict["file_beam"],
|
|
1088
1092
|
found_dict["file_pol"],
|
|
1089
1093
|
s1_title_suffix,
|
|
1090
|
-
S1_IMG_NB_PER_POLAR.get(
|
|
1094
|
+
S1_IMG_NB_PER_POLAR.get(polarization_mode, {}).get(
|
|
1091
1095
|
found_dict["file_pol"].upper(), 1
|
|
1092
1096
|
),
|
|
1093
1097
|
found_dict["file_ext"],
|
|
@@ -1112,7 +1116,7 @@ class AwsDownload(Download):
|
|
|
1112
1116
|
def download_all(
|
|
1113
1117
|
self,
|
|
1114
1118
|
products: SearchResult,
|
|
1115
|
-
auth: Optional[Union[AuthBase,
|
|
1119
|
+
auth: Optional[Union[AuthBase, S3ServiceResource]] = None,
|
|
1116
1120
|
downloaded_callback: Optional[DownloadedCallback] = None,
|
|
1117
1121
|
progress_callback: Optional[ProgressCallback] = None,
|
|
1118
1122
|
wait: float = DEFAULT_DOWNLOAD_WAIT,
|