eodag 3.0.0b1__py3-none-any.whl → 3.0.0b3__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- eodag/__init__.py +6 -8
- eodag/api/core.py +119 -171
- eodag/api/product/__init__.py +10 -4
- eodag/api/product/_assets.py +52 -14
- eodag/api/product/_product.py +59 -30
- eodag/api/product/drivers/__init__.py +7 -2
- eodag/api/product/drivers/base.py +0 -3
- eodag/api/product/metadata_mapping.py +0 -28
- eodag/api/search_result.py +31 -9
- eodag/config.py +45 -41
- eodag/plugins/apis/base.py +3 -3
- eodag/plugins/apis/ecmwf.py +2 -3
- eodag/plugins/apis/usgs.py +43 -14
- eodag/plugins/authentication/aws_auth.py +11 -2
- eodag/plugins/authentication/openid_connect.py +5 -4
- eodag/plugins/authentication/token.py +2 -1
- eodag/plugins/crunch/base.py +3 -1
- eodag/plugins/crunch/filter_date.py +3 -9
- eodag/plugins/crunch/filter_latest_intersect.py +0 -3
- eodag/plugins/crunch/filter_latest_tpl_name.py +1 -4
- eodag/plugins/crunch/filter_overlap.py +4 -8
- eodag/plugins/crunch/filter_property.py +5 -11
- eodag/plugins/download/aws.py +46 -78
- eodag/plugins/download/base.py +27 -68
- eodag/plugins/download/http.py +48 -57
- eodag/plugins/download/s3rest.py +17 -25
- eodag/plugins/manager.py +6 -18
- eodag/plugins/search/__init__.py +9 -9
- eodag/plugins/search/base.py +7 -26
- eodag/plugins/search/build_search_result.py +0 -13
- eodag/plugins/search/cop_marine.py +1 -3
- eodag/plugins/search/creodias_s3.py +0 -3
- eodag/plugins/search/data_request_search.py +10 -5
- eodag/plugins/search/qssearch.py +95 -53
- eodag/plugins/search/static_stac_search.py +6 -3
- eodag/resources/ext_product_types.json +1 -1
- eodag/resources/product_types.yml +24 -0
- eodag/resources/providers.yml +198 -154
- eodag/resources/user_conf_template.yml +27 -27
- eodag/rest/core.py +11 -43
- eodag/rest/server.py +1 -6
- eodag/rest/stac.py +13 -87
- eodag/rest/types/eodag_search.py +4 -7
- eodag/rest/types/queryables.py +4 -12
- eodag/rest/types/stac_search.py +7 -11
- eodag/rest/utils/rfc3339.py +0 -1
- eodag/types/__init__.py +9 -3
- eodag/types/download_args.py +14 -5
- eodag/types/search_args.py +7 -8
- eodag/types/whoosh.py +0 -2
- eodag/utils/__init__.py +20 -79
- eodag/utils/constraints.py +0 -8
- eodag/utils/import_system.py +0 -4
- eodag/utils/logging.py +0 -3
- eodag/utils/notebook.py +4 -4
- eodag/utils/repr.py +113 -0
- eodag/utils/requests.py +12 -20
- eodag/utils/rest.py +0 -4
- eodag/utils/stac_reader.py +2 -14
- {eodag-3.0.0b1.dist-info → eodag-3.0.0b3.dist-info}/METADATA +33 -14
- eodag-3.0.0b3.dist-info/RECORD +110 -0
- {eodag-3.0.0b1.dist-info → eodag-3.0.0b3.dist-info}/WHEEL +1 -1
- eodag-3.0.0b1.dist-info/RECORD +0 -109
- {eodag-3.0.0b1.dist-info → eodag-3.0.0b3.dist-info}/LICENSE +0 -0
- {eodag-3.0.0b1.dist-info → eodag-3.0.0b3.dist-info}/entry_points.txt +0 -0
- {eodag-3.0.0b1.dist-info → eodag-3.0.0b3.dist-info}/top_level.txt +0 -0
eodag/config.py
CHANGED
|
@@ -47,6 +47,7 @@ from pkg_resources import resource_filename
|
|
|
47
47
|
from requests.auth import AuthBase
|
|
48
48
|
from typing_extensions import Doc
|
|
49
49
|
|
|
50
|
+
from eodag.api.product.metadata_mapping import mtd_cfg_as_conversion_and_querypath
|
|
50
51
|
from eodag.utils import (
|
|
51
52
|
HTTP_REQ_TIMEOUT,
|
|
52
53
|
USER_AGENT,
|
|
@@ -110,22 +111,14 @@ class ProviderConfig(yaml.YAMLObject):
|
|
|
110
111
|
"""Representation of eodag configuration.
|
|
111
112
|
|
|
112
113
|
:param name: The name of the provider
|
|
113
|
-
:type name: str
|
|
114
114
|
:param priority: (optional) The priority of the provider while searching a product.
|
|
115
115
|
Lower value means lower priority. (Default: 0)
|
|
116
|
-
:type priority: int
|
|
117
116
|
:param api: (optional) The configuration of a plugin of type Api
|
|
118
|
-
:type api: :class:`~eodag.config.PluginConfig`
|
|
119
117
|
:param search: (optional) The configuration of a plugin of type Search
|
|
120
|
-
:type search: :class:`~eodag.config.PluginConfig`
|
|
121
118
|
:param products: (optional) The products types supported by the provider
|
|
122
|
-
:type products: dict
|
|
123
119
|
:param download: (optional) The configuration of a plugin of type Download
|
|
124
|
-
:type download: :class:`~eodag.config.PluginConfig`
|
|
125
120
|
:param auth: (optional) The configuration of a plugin of type Authentication
|
|
126
|
-
:type auth: :class:`~eodag.config.PluginConfig`
|
|
127
121
|
:param kwargs: Additional configuration variables for this provider
|
|
128
|
-
:type kwargs: Any
|
|
129
122
|
"""
|
|
130
123
|
|
|
131
124
|
name: str
|
|
@@ -171,7 +164,6 @@ class ProviderConfig(yaml.YAMLObject):
|
|
|
171
164
|
"""Validate a :class:`~eodag.config.ProviderConfig`
|
|
172
165
|
|
|
173
166
|
:param config_keys: The configurations keys to validate
|
|
174
|
-
:type config_keys: dict
|
|
175
167
|
"""
|
|
176
168
|
if "name" not in config_keys:
|
|
177
169
|
raise ValidationError("Provider config must have name key")
|
|
@@ -189,7 +181,6 @@ class ProviderConfig(yaml.YAMLObject):
|
|
|
189
181
|
"""Update the configuration parameters with values from `mapping`
|
|
190
182
|
|
|
191
183
|
:param mapping: The mapping from which to override configuration parameters
|
|
192
|
-
:type mapping: dict
|
|
193
184
|
"""
|
|
194
185
|
if mapping is None:
|
|
195
186
|
mapping = {}
|
|
@@ -215,12 +206,9 @@ class PluginConfig(yaml.YAMLObject):
|
|
|
215
206
|
"""Representation of a plugin config
|
|
216
207
|
|
|
217
208
|
:param name: The name of the plugin class to use to instantiate the plugin object
|
|
218
|
-
:type name: str
|
|
219
209
|
:param metadata_mapping: (optional) The mapping between eodag metadata and
|
|
220
210
|
the plugin specific metadata
|
|
221
|
-
:type metadata_mapping: dict
|
|
222
211
|
:param free_params: (optional) Additional configuration parameters
|
|
223
|
-
:type free_params: dict
|
|
224
212
|
"""
|
|
225
213
|
|
|
226
214
|
class Pagination(TypedDict):
|
|
@@ -246,6 +234,14 @@ class PluginConfig(yaml.YAMLObject):
|
|
|
246
234
|
sort_order_mapping: Dict[Literal["ascending", "descending"], str]
|
|
247
235
|
max_sort_params: Annotated[int, Gt(0)]
|
|
248
236
|
|
|
237
|
+
class DiscoverMetadata(TypedDict):
|
|
238
|
+
"""Configuration for metadata discovery"""
|
|
239
|
+
|
|
240
|
+
auto_discovery: bool
|
|
241
|
+
metadata_pattern: str
|
|
242
|
+
search_param: str
|
|
243
|
+
metadata_path: str
|
|
244
|
+
|
|
249
245
|
class OrderOnResponse(TypedDict):
|
|
250
246
|
"""Configuration for order on-response during download"""
|
|
251
247
|
|
|
@@ -319,7 +315,7 @@ class PluginConfig(yaml.YAMLObject):
|
|
|
319
315
|
pagination: PluginConfig.Pagination
|
|
320
316
|
sort: PluginConfig.Sort
|
|
321
317
|
query_params_key: str
|
|
322
|
-
discover_metadata:
|
|
318
|
+
discover_metadata: PluginConfig.DiscoverMetadata
|
|
323
319
|
discover_product_types: Dict[str, Any]
|
|
324
320
|
discover_queryables: Dict[str, Any]
|
|
325
321
|
metadata_mapping: Dict[str, Union[str, List[str]]]
|
|
@@ -343,9 +339,9 @@ class PluginConfig(yaml.YAMLObject):
|
|
|
343
339
|
|
|
344
340
|
# download -------------------------------------------------------------------------
|
|
345
341
|
base_uri: str
|
|
346
|
-
|
|
342
|
+
output_dir: str
|
|
347
343
|
extract: bool
|
|
348
|
-
|
|
344
|
+
output_extension: str
|
|
349
345
|
order_enabled: bool # HTTPDownload
|
|
350
346
|
order_method: str # HTTPDownload
|
|
351
347
|
order_headers: Dict[str, str] # HTTPDownload
|
|
@@ -419,7 +415,6 @@ class PluginConfig(yaml.YAMLObject):
|
|
|
419
415
|
"""Update the configuration parameters with values from `mapping`
|
|
420
416
|
|
|
421
417
|
:param mapping: The mapping from which to override configuration parameters
|
|
422
|
-
:type mapping: dict
|
|
423
418
|
"""
|
|
424
419
|
if mapping is None:
|
|
425
420
|
mapping = {}
|
|
@@ -435,7 +430,6 @@ def load_default_config() -> Dict[str, ProviderConfig]:
|
|
|
435
430
|
variable if exists.
|
|
436
431
|
|
|
437
432
|
:returns: The default provider's configuration
|
|
438
|
-
:rtype: dict
|
|
439
433
|
"""
|
|
440
434
|
eodag_providers_cfg_file = os.getenv(
|
|
441
435
|
"EODAG_PROVIDERS_CFG_FILE"
|
|
@@ -447,9 +441,7 @@ def load_config(config_path: str) -> Dict[str, ProviderConfig]:
|
|
|
447
441
|
"""Load the providers configuration into a dictionnary from a given file
|
|
448
442
|
|
|
449
443
|
:param config_path: The path to the provider config file
|
|
450
|
-
:type config_path: str
|
|
451
444
|
:returns: The default provider's configuration
|
|
452
|
-
:rtype: dict
|
|
453
445
|
"""
|
|
454
446
|
logger.debug("Loading configuration from %s", config_path)
|
|
455
447
|
config: Dict[str, ProviderConfig] = {}
|
|
@@ -475,17 +467,15 @@ def provider_config_init(
|
|
|
475
467
|
"""Applies some default values to provider config
|
|
476
468
|
|
|
477
469
|
:param provider_config: An eodag provider configuration
|
|
478
|
-
:type provider_config: :class:`~eodag.config.ProviderConfig`
|
|
479
470
|
:param stac_search_default_conf: default conf to overwrite with provider_config if STAC
|
|
480
|
-
:type stac_search_default_conf: dict
|
|
481
471
|
"""
|
|
482
|
-
# For the provider, set the default
|
|
472
|
+
# For the provider, set the default output_dir of its download plugin
|
|
483
473
|
# as tempdir in a portable way
|
|
484
474
|
for param_name in ("download", "api"):
|
|
485
475
|
if param_name in vars(provider_config):
|
|
486
476
|
param_value = getattr(provider_config, param_name)
|
|
487
|
-
if not getattr(param_value, "
|
|
488
|
-
param_value.
|
|
477
|
+
if not getattr(param_value, "output_dir", None):
|
|
478
|
+
param_value.output_dir = tempfile.gettempdir()
|
|
489
479
|
if not getattr(param_value, "delete_archive", None):
|
|
490
480
|
param_value.delete_archive = True
|
|
491
481
|
|
|
@@ -514,9 +504,7 @@ def override_config_from_file(config: Dict[str, Any], file_path: str) -> None:
|
|
|
514
504
|
"""Override a configuration with the values in a file
|
|
515
505
|
|
|
516
506
|
:param config: An eodag providers configuration dictionary
|
|
517
|
-
:type config: dict
|
|
518
507
|
:param file_path: The path to the file from where the new values will be read
|
|
519
|
-
:type file_path: str
|
|
520
508
|
"""
|
|
521
509
|
logger.info("Loading user configuration from: %s", os.path.abspath(file_path))
|
|
522
510
|
with open(os.path.abspath(os.path.realpath(file_path)), "r") as fh:
|
|
@@ -534,7 +522,6 @@ def override_config_from_env(config: Dict[str, Any]) -> None:
|
|
|
534
522
|
"""Override a configuration with environment variables values
|
|
535
523
|
|
|
536
524
|
:param config: An eodag providers configuration dictionary
|
|
537
|
-
:type config: dict
|
|
538
525
|
"""
|
|
539
526
|
|
|
540
527
|
def build_mapping_from_env(
|
|
@@ -554,11 +541,8 @@ def override_config_from_env(config: Dict[str, Any]) -> None:
|
|
|
554
541
|
}
|
|
555
542
|
|
|
556
543
|
:param env_var: The environment variable to be transformed into a dictionary
|
|
557
|
-
:type env_var: str
|
|
558
544
|
:param env_value: The value from environment variable
|
|
559
|
-
:type env_value: str
|
|
560
545
|
:param mapping: The mapping in which the value will be created
|
|
561
|
-
:type mapping: dict
|
|
562
546
|
"""
|
|
563
547
|
parts = env_var.split("__")
|
|
564
548
|
iter_parts = iter(parts)
|
|
@@ -610,11 +594,39 @@ def override_config_from_mapping(
|
|
|
610
594
|
"""Override a configuration with the values in a mapping
|
|
611
595
|
|
|
612
596
|
:param config: An eodag providers configuration dictionary
|
|
613
|
-
:type config: dict
|
|
614
597
|
:param mapping: The mapping containing the values to be overriden
|
|
615
|
-
:type mapping: dict
|
|
616
598
|
"""
|
|
617
599
|
for provider, new_conf in mapping.items():
|
|
600
|
+
# check if metada-mapping as already been built as jsonpath in providers_config
|
|
601
|
+
if not isinstance(new_conf, dict):
|
|
602
|
+
continue
|
|
603
|
+
new_conf_search = new_conf.get("search", {}) or {}
|
|
604
|
+
new_conf_api = new_conf.get("api", {}) or {}
|
|
605
|
+
if provider in config and "metadata_mapping" in {
|
|
606
|
+
**new_conf_search,
|
|
607
|
+
**new_conf_api,
|
|
608
|
+
}:
|
|
609
|
+
search_plugin_key = (
|
|
610
|
+
"search" if "metadata_mapping" in new_conf_search else "api"
|
|
611
|
+
)
|
|
612
|
+
# get some already configured value
|
|
613
|
+
configured_metadata_mapping = getattr(
|
|
614
|
+
config[provider], search_plugin_key
|
|
615
|
+
).metadata_mapping
|
|
616
|
+
some_configured_value = next(iter(configured_metadata_mapping.values()))
|
|
617
|
+
# check if the configured value has already been built as jsonpath
|
|
618
|
+
if (
|
|
619
|
+
isinstance(some_configured_value, list)
|
|
620
|
+
and isinstance(some_configured_value[1], tuple)
|
|
621
|
+
or isinstance(some_configured_value, tuple)
|
|
622
|
+
):
|
|
623
|
+
# also build as jsonpath the incoming conf
|
|
624
|
+
mtd_cfg_as_conversion_and_querypath(
|
|
625
|
+
deepcopy(mapping[provider][search_plugin_key]["metadata_mapping"]),
|
|
626
|
+
mapping[provider][search_plugin_key]["metadata_mapping"],
|
|
627
|
+
)
|
|
628
|
+
|
|
629
|
+
# try overriding conf
|
|
618
630
|
old_conf: Optional[Dict[str, Any]] = config.get(provider)
|
|
619
631
|
if old_conf is not None:
|
|
620
632
|
old_conf.update(new_conf)
|
|
@@ -640,9 +652,7 @@ def merge_configs(config: Dict[str, Any], other_config: Dict[str, Any]) -> None:
|
|
|
640
652
|
"""Override a configuration with the values of another configuration
|
|
641
653
|
|
|
642
654
|
:param config: An eodag providers configuration dictionary
|
|
643
|
-
:type config: dict
|
|
644
655
|
:param other_config: An eodag providers configuration dictionary
|
|
645
|
-
:type other_config: dict
|
|
646
656
|
"""
|
|
647
657
|
# configs union with other_config values as default
|
|
648
658
|
other_config = dict(config, **other_config)
|
|
@@ -674,7 +684,6 @@ def load_yml_config(yml_path: str) -> Dict[Any, Any]:
|
|
|
674
684
|
"""Load a conf dictionnary from given yml absolute path
|
|
675
685
|
|
|
676
686
|
:returns: The yml configuration file
|
|
677
|
-
:rtype: dict
|
|
678
687
|
"""
|
|
679
688
|
config = SimpleYamlProxyConfig(yml_path)
|
|
680
689
|
return dict_items_recursive_apply(config.source, string_to_jsonpath)
|
|
@@ -684,7 +693,6 @@ def load_stac_config() -> Dict[str, Any]:
|
|
|
684
693
|
"""Load the stac configuration into a dictionnary
|
|
685
694
|
|
|
686
695
|
:returns: The stac configuration
|
|
687
|
-
:rtype: dict
|
|
688
696
|
"""
|
|
689
697
|
return load_yml_config(
|
|
690
698
|
resource_filename("eodag", os.path.join("resources/", "stac.yml"))
|
|
@@ -695,7 +703,6 @@ def load_stac_api_config() -> Dict[str, Any]:
|
|
|
695
703
|
"""Load the stac API configuration into a dictionnary
|
|
696
704
|
|
|
697
705
|
:returns: The stac API configuration
|
|
698
|
-
:rtype: dict
|
|
699
706
|
"""
|
|
700
707
|
return load_yml_config(
|
|
701
708
|
resource_filename("eodag", os.path.join("resources/", "stac_api.yml"))
|
|
@@ -706,7 +713,6 @@ def load_stac_provider_config() -> Dict[str, Any]:
|
|
|
706
713
|
"""Load the stac provider configuration into a dictionnary
|
|
707
714
|
|
|
708
715
|
:returns: The stac provider configuration
|
|
709
|
-
:rtype: dict
|
|
710
716
|
"""
|
|
711
717
|
return SimpleYamlProxyConfig(
|
|
712
718
|
resource_filename("eodag", os.path.join("resources/", "stac_provider.yml"))
|
|
@@ -719,9 +725,7 @@ def get_ext_product_types_conf(
|
|
|
719
725
|
"""Read external product types conf
|
|
720
726
|
|
|
721
727
|
:param conf_uri: URI to local or remote configuration file
|
|
722
|
-
:type conf_uri: str
|
|
723
728
|
:returns: The external product types configuration
|
|
724
|
-
:rtype: dict
|
|
725
729
|
"""
|
|
726
730
|
logger.info("Fetching external product types from %s", conf_uri)
|
|
727
731
|
if conf_uri.lower().startswith("http"):
|
eodag/plugins/apis/base.py
CHANGED
|
@@ -33,17 +33,17 @@ class Api(Search, Download):
|
|
|
33
33
|
|
|
34
34
|
The download methods must:
|
|
35
35
|
|
|
36
|
-
- download data in the ``
|
|
36
|
+
- download data in the ``output_dir`` folder defined in the plugin's
|
|
37
37
|
configuration or passed through kwargs
|
|
38
38
|
- extract products from their archive (if relevant) if ``extract`` is set to True
|
|
39
39
|
(True by default)
|
|
40
|
-
- save a product in an archive/directory (in ``
|
|
40
|
+
- save a product in an archive/directory (in ``output_dir``) whose name must be
|
|
41
41
|
the product's ``title`` property
|
|
42
42
|
- update the product's ``location`` attribute once its data is downloaded (and
|
|
43
43
|
eventually after it's extracted) to the product's location given as a file URI
|
|
44
44
|
(e.g. 'file:///tmp/product_folder' on Linux or
|
|
45
45
|
'file:///C:/Users/username/AppData/LOcal/Temp' on Windows)
|
|
46
|
-
- save a *record* file in the directory ``
|
|
46
|
+
- save a *record* file in the directory ``output_dir/.downloaded`` whose name
|
|
47
47
|
is built on the MD5 hash of the product's ``product_type`` and ``properties['id']``
|
|
48
48
|
attributes (``hashlib.md5((product.product_type+"-"+product.properties['id']).encode("utf-8")).hexdigest()``)
|
|
49
49
|
and whose content is the product's ``remote_location`` attribute itself.
|
eodag/plugins/apis/ecmwf.py
CHANGED
|
@@ -128,7 +128,6 @@ class EcmwfApi(Api, BuildPostSearchResult):
|
|
|
128
128
|
"""Check credentials and returns information needed for auth
|
|
129
129
|
|
|
130
130
|
:returns: {key, url, email} dictionary
|
|
131
|
-
:rtype: dict
|
|
132
131
|
:raises: :class:`~eodag.utils.exceptions.AuthenticationError`
|
|
133
132
|
"""
|
|
134
133
|
# Get credentials from eodag or using ecmwf conf
|
|
@@ -165,8 +164,8 @@ class EcmwfApi(Api, BuildPostSearchResult):
|
|
|
165
164
|
"""Download data from ECMWF MARS"""
|
|
166
165
|
product_format = product.properties.get("format", "grib")
|
|
167
166
|
product_extension = ECMWF_MARS_KNOWN_FORMATS.get(product_format, product_format)
|
|
168
|
-
kwargs["
|
|
169
|
-
"
|
|
167
|
+
kwargs["output_extension"] = kwargs.get(
|
|
168
|
+
"output_extension", f".{product_extension}"
|
|
170
169
|
)
|
|
171
170
|
|
|
172
171
|
# Prepare download
|
eodag/plugins/apis/usgs.py
CHANGED
|
@@ -128,13 +128,14 @@ class UsgsApi(Api):
|
|
|
128
128
|
raise NoMatchingProductType(
|
|
129
129
|
"Cannot search on USGS without productType specified"
|
|
130
130
|
)
|
|
131
|
-
if kwargs.get("
|
|
131
|
+
if kwargs.get("sort_by"):
|
|
132
132
|
raise ValidationError("USGS does not support sorting feature")
|
|
133
133
|
|
|
134
134
|
self.authenticate()
|
|
135
135
|
|
|
136
136
|
product_type_def_params = self.config.products.get( # type: ignore
|
|
137
|
-
product_type,
|
|
137
|
+
product_type,
|
|
138
|
+
self.config.products[GENERIC_PRODUCT_TYPE], # type: ignore
|
|
138
139
|
)
|
|
139
140
|
usgs_dataset = format_dict_items(product_type_def_params, **kwargs)["dataset"]
|
|
140
141
|
start_date = kwargs.pop("startTimeFromAscendingNode", None)
|
|
@@ -172,11 +173,39 @@ class UsgsApi(Api):
|
|
|
172
173
|
max_results=items_per_page,
|
|
173
174
|
starting_number=(1 + (page - 1) * items_per_page),
|
|
174
175
|
)
|
|
175
|
-
logger.info(
|
|
176
|
-
f"Sending search request for {usgs_dataset} with {api_search_kwargs}"
|
|
177
|
-
)
|
|
178
176
|
|
|
179
|
-
|
|
177
|
+
# search by id
|
|
178
|
+
if searched_id := kwargs.get("id"):
|
|
179
|
+
dataset_filters = api.dataset_filters(usgs_dataset)
|
|
180
|
+
# ip pattern set as parameter queryable (first element of param conf list)
|
|
181
|
+
id_pattern = self.config.metadata_mapping["id"][0]
|
|
182
|
+
# loop on matching dataset_filters until one returns expected results
|
|
183
|
+
for dataset_filter in dataset_filters["data"]:
|
|
184
|
+
if id_pattern in dataset_filter["searchSql"]:
|
|
185
|
+
logger.debug(
|
|
186
|
+
f"Try using {dataset_filter['searchSql']} dataset filter to search by id on {usgs_dataset}"
|
|
187
|
+
)
|
|
188
|
+
full_api_search_kwargs = {
|
|
189
|
+
"where": {
|
|
190
|
+
"filter_id": dataset_filter["id"],
|
|
191
|
+
"value": searched_id,
|
|
192
|
+
},
|
|
193
|
+
**api_search_kwargs,
|
|
194
|
+
}
|
|
195
|
+
logger.info(
|
|
196
|
+
f"Sending search request for {usgs_dataset} with {full_api_search_kwargs}"
|
|
197
|
+
)
|
|
198
|
+
results = api.scene_search(
|
|
199
|
+
usgs_dataset, **full_api_search_kwargs
|
|
200
|
+
)
|
|
201
|
+
if len(results["data"]["results"]) == 1:
|
|
202
|
+
# search by id using this dataset_filter succeeded
|
|
203
|
+
break
|
|
204
|
+
else:
|
|
205
|
+
logger.info(
|
|
206
|
+
f"Sending search request for {usgs_dataset} with {api_search_kwargs}"
|
|
207
|
+
)
|
|
208
|
+
results = api.scene_search(usgs_dataset, **api_search_kwargs)
|
|
180
209
|
|
|
181
210
|
# update results with storage info from download_options()
|
|
182
211
|
results_by_entity_id = {
|
|
@@ -257,13 +286,13 @@ class UsgsApi(Api):
|
|
|
257
286
|
)
|
|
258
287
|
progress_callback = ProgressCallback(disable=True)
|
|
259
288
|
|
|
260
|
-
|
|
289
|
+
output_extension = cast(
|
|
261
290
|
str,
|
|
262
291
|
self.config.products.get( # type: ignore
|
|
263
292
|
product.product_type, self.config.products[GENERIC_PRODUCT_TYPE] # type: ignore
|
|
264
|
-
).get("
|
|
293
|
+
).get("output_extension", ".tar.gz"),
|
|
265
294
|
)
|
|
266
|
-
kwargs["
|
|
295
|
+
kwargs["output_extension"] = kwargs.get("output_extension", output_extension)
|
|
267
296
|
|
|
268
297
|
fs_path, record_filename = self._prepare_download(
|
|
269
298
|
product,
|
|
@@ -375,8 +404,8 @@ class UsgsApi(Api):
|
|
|
375
404
|
|
|
376
405
|
# Check downloaded file format
|
|
377
406
|
if (
|
|
378
|
-
kwargs["
|
|
379
|
-
) or (kwargs["
|
|
407
|
+
kwargs["output_extension"] == ".tar.gz" and tarfile.is_tarfile(fs_path)
|
|
408
|
+
) or (kwargs["output_extension"] == ".zip" and zipfile.is_zipfile(fs_path)):
|
|
380
409
|
product_path = self._finalize(
|
|
381
410
|
fs_path,
|
|
382
411
|
progress_callback=progress_callback,
|
|
@@ -388,7 +417,7 @@ class UsgsApi(Api):
|
|
|
388
417
|
logger.info(
|
|
389
418
|
"Downloaded product detected as a tar File, but was was expected to be a zip file"
|
|
390
419
|
)
|
|
391
|
-
new_fs_path = fs_path[: fs_path.index(
|
|
420
|
+
new_fs_path = fs_path[: fs_path.index(output_extension)] + ".tar.gz"
|
|
392
421
|
shutil.move(fs_path, new_fs_path)
|
|
393
422
|
product.location = path_to_uri(new_fs_path)
|
|
394
423
|
return new_fs_path
|
|
@@ -396,7 +425,7 @@ class UsgsApi(Api):
|
|
|
396
425
|
logger.info(
|
|
397
426
|
"Downloaded product detected as a zip File, but was was expected to be a tar file"
|
|
398
427
|
)
|
|
399
|
-
new_fs_path = fs_path[: fs_path.index(
|
|
428
|
+
new_fs_path = fs_path[: fs_path.index(output_extension)] + ".zip"
|
|
400
429
|
shutil.move(fs_path, new_fs_path)
|
|
401
430
|
product.location = path_to_uri(new_fs_path)
|
|
402
431
|
return new_fs_path
|
|
@@ -404,7 +433,7 @@ class UsgsApi(Api):
|
|
|
404
433
|
logger.warning(
|
|
405
434
|
"Downloaded product is not a tar or a zip File. Please check its file type before using it"
|
|
406
435
|
)
|
|
407
|
-
new_fs_path = fs_path[: fs_path.index(
|
|
436
|
+
new_fs_path = fs_path[: fs_path.index(output_extension)]
|
|
408
437
|
shutil.move(fs_path, new_fs_path)
|
|
409
438
|
product.location = path_to_uri(new_fs_path)
|
|
410
439
|
return new_fs_path
|
|
@@ -35,6 +35,7 @@ class AwsAuth(Authentication):
|
|
|
35
35
|
- auth anonymously using no-sign-request
|
|
36
36
|
- auth using ``aws_profile``
|
|
37
37
|
- auth using ``aws_access_key_id`` and ``aws_secret_access_key``
|
|
38
|
+
(optionally ``aws_session_token``)
|
|
38
39
|
- auth using current environment (AWS environment variables and/or ``~/aws/*``),
|
|
39
40
|
will be skipped if AWS credentials are filled in eodag conf
|
|
40
41
|
"""
|
|
@@ -45,13 +46,13 @@ class AwsAuth(Authentication):
|
|
|
45
46
|
super(AwsAuth, self).__init__(provider, config)
|
|
46
47
|
self.aws_access_key_id = None
|
|
47
48
|
self.aws_secret_access_key = None
|
|
49
|
+
self.aws_session_token = None
|
|
48
50
|
self.profile_name = None
|
|
49
51
|
|
|
50
52
|
def authenticate(self) -> Dict[str, str]:
|
|
51
53
|
"""Authenticate
|
|
52
54
|
|
|
53
55
|
:returns: dict containing AWS/boto3 non-empty credentials
|
|
54
|
-
:rtype: dict
|
|
55
56
|
"""
|
|
56
57
|
credentials = getattr(self.config, "credentials", {}) or {}
|
|
57
58
|
self.aws_access_key_id = credentials.get(
|
|
@@ -60,7 +61,15 @@ class AwsAuth(Authentication):
|
|
|
60
61
|
self.aws_secret_access_key = credentials.get(
|
|
61
62
|
"aws_secret_access_key", self.aws_secret_access_key
|
|
62
63
|
)
|
|
64
|
+
self.aws_session_token = credentials.get(
|
|
65
|
+
"aws_session_token", self.aws_session_token
|
|
66
|
+
)
|
|
63
67
|
self.profile_name = credentials.get("aws_profile", self.profile_name)
|
|
64
68
|
|
|
65
|
-
auth_keys = [
|
|
69
|
+
auth_keys = [
|
|
70
|
+
"aws_access_key_id",
|
|
71
|
+
"aws_secret_access_key",
|
|
72
|
+
"aws_session_token",
|
|
73
|
+
"profile_name",
|
|
74
|
+
]
|
|
66
75
|
return {k: getattr(self, k) for k in auth_keys if getattr(self, k)}
|
|
@@ -482,7 +482,7 @@ class OIDCAuthorizationCodeFlowAuth(OIDCRefreshTokenBase):
|
|
|
482
482
|
if not match:
|
|
483
483
|
return value
|
|
484
484
|
value_from_xpath = form_element.xpath(
|
|
485
|
-
|
|
485
|
+
match.groupdict("xpath_value")["xpath_value"]
|
|
486
486
|
)
|
|
487
487
|
if len(value_from_xpath) == 1:
|
|
488
488
|
return value_from_xpath[0]
|
|
@@ -512,10 +512,11 @@ class CodeAuthorizedAuth(AuthBase):
|
|
|
512
512
|
def __call__(self, request: PreparedRequest) -> PreparedRequest:
|
|
513
513
|
"""Perform the actual authentication"""
|
|
514
514
|
if self.where == "qs":
|
|
515
|
-
parts = urlparse(request.url)
|
|
515
|
+
parts = urlparse(str(request.url))
|
|
516
516
|
query_dict = parse_qs(parts.query)
|
|
517
|
-
|
|
518
|
-
|
|
517
|
+
if self.key is not None:
|
|
518
|
+
query_dict.update({self.key: [self.token]})
|
|
519
|
+
url_without_args = parts._replace(query="").geturl()
|
|
519
520
|
|
|
520
521
|
request.prepare_url(url_without_args, query_dict)
|
|
521
522
|
|
|
@@ -197,7 +197,8 @@ class RequestsTokenAuth(AuthBase):
|
|
|
197
197
|
if self.where == "qs":
|
|
198
198
|
parts = urlparse(str(request.url))
|
|
199
199
|
qs = parse_qs(parts.query)
|
|
200
|
-
|
|
200
|
+
if self.qs_key is not None:
|
|
201
|
+
qs[self.qs_key] = [self.token]
|
|
201
202
|
request.url = urlunparse(
|
|
202
203
|
(
|
|
203
204
|
parts.scheme,
|
eodag/plugins/crunch/base.py
CHANGED
|
@@ -19,6 +19,7 @@ from __future__ import annotations
|
|
|
19
19
|
|
|
20
20
|
from typing import TYPE_CHECKING, Any, Dict, List, Optional
|
|
21
21
|
|
|
22
|
+
from eodag.config import PluginConfig
|
|
22
23
|
from eodag.plugins.base import PluginTopic
|
|
23
24
|
|
|
24
25
|
if TYPE_CHECKING:
|
|
@@ -29,7 +30,8 @@ class Crunch(PluginTopic):
|
|
|
29
30
|
"""Base cruncher"""
|
|
30
31
|
|
|
31
32
|
def __init__(self, config: Optional[Dict[str, Any]]) -> None:
|
|
32
|
-
self.config =
|
|
33
|
+
self.config = PluginConfig()
|
|
34
|
+
self.config.__dict__ = config if config is not None else {}
|
|
33
35
|
|
|
34
36
|
def proceed(
|
|
35
37
|
self, products: List[EOProduct], **search_params: Any
|
|
@@ -21,7 +21,7 @@ import datetime
|
|
|
21
21
|
import logging
|
|
22
22
|
import time
|
|
23
23
|
from datetime import datetime as dt
|
|
24
|
-
from typing import TYPE_CHECKING, Any,
|
|
24
|
+
from typing import TYPE_CHECKING, Any, List
|
|
25
25
|
|
|
26
26
|
import dateutil.parser
|
|
27
27
|
from dateutil import tz
|
|
@@ -41,12 +41,8 @@ class FilterDate(Crunch):
|
|
|
41
41
|
|
|
42
42
|
- `start`: (optional) start sensing time in iso format
|
|
43
43
|
- `end`: (optional) end sensing time in iso format
|
|
44
|
-
|
|
45
|
-
:type config: dict
|
|
46
44
|
"""
|
|
47
45
|
|
|
48
|
-
config: Dict[str, str]
|
|
49
|
-
|
|
50
46
|
@staticmethod
|
|
51
47
|
def sort_product_by_start_date(product: EOProduct) -> dt:
|
|
52
48
|
"""Get product start date"""
|
|
@@ -63,16 +59,14 @@ class FilterDate(Crunch):
|
|
|
63
59
|
"""Execute crunch: Filter products between start and end dates.
|
|
64
60
|
|
|
65
61
|
:param products: A list of products resulting from a search
|
|
66
|
-
:type products: list(:class:`~eodag.api.product._product.EOProduct`)
|
|
67
62
|
:returns: The filtered products
|
|
68
|
-
:rtype: list(:class:`~eodag.api.product._product.EOProduct`)
|
|
69
63
|
"""
|
|
70
64
|
logger.debug("Start filtering by date")
|
|
71
65
|
if not products:
|
|
72
66
|
return []
|
|
73
67
|
|
|
74
68
|
# filter start date
|
|
75
|
-
filter_start_str = self.config.get("start", None)
|
|
69
|
+
filter_start_str = self.config.__dict__.get("start", None)
|
|
76
70
|
if filter_start_str:
|
|
77
71
|
filter_start = dateutil.parser.parse(filter_start_str)
|
|
78
72
|
if not filter_start.tzinfo:
|
|
@@ -81,7 +75,7 @@ class FilterDate(Crunch):
|
|
|
81
75
|
filter_start = None
|
|
82
76
|
|
|
83
77
|
# filter end date
|
|
84
|
-
filter_end_str = self.config.get("end", None)
|
|
78
|
+
filter_end_str = self.config.__dict__.get("end", None)
|
|
85
79
|
if filter_end_str:
|
|
86
80
|
filter_end = dateutil.parser.parse(filter_end_str)
|
|
87
81
|
if not filter_end.tzinfo:
|
|
@@ -59,12 +59,9 @@ class FilterLatestIntersect(Crunch):
|
|
|
59
59
|
Filter latest products (the ones with a the highest start date) that intersect search extent.
|
|
60
60
|
|
|
61
61
|
:param products: A list of products resulting from a search
|
|
62
|
-
:type products: list(:class:`~eodag.api.product._product.EOProduct`)
|
|
63
62
|
:param search_params: Search criteria that must contain `geometry` (dict)
|
|
64
63
|
or search `geom` (:class:`shapely.geometry.base.BaseGeometry`) argument will be used
|
|
65
|
-
:type search_params: dict
|
|
66
64
|
:returns: The filtered products
|
|
67
|
-
:rtype: list(:class:`~eodag.api.product._product.EOProduct`)
|
|
68
65
|
"""
|
|
69
66
|
logger.debug("Start filtering for latest products")
|
|
70
67
|
if not products:
|
|
@@ -26,6 +26,7 @@ from eodag.utils.exceptions import ValidationError
|
|
|
26
26
|
|
|
27
27
|
if TYPE_CHECKING:
|
|
28
28
|
from eodag.api.product import EOProduct
|
|
29
|
+
|
|
29
30
|
logger = logging.getLogger("eodag.crunch.latest_tpl_name")
|
|
30
31
|
|
|
31
32
|
|
|
@@ -37,8 +38,6 @@ class FilterLatestByName(Crunch):
|
|
|
37
38
|
:param config: Crunch configuration, must contain :
|
|
38
39
|
|
|
39
40
|
- `name_pattern` : product name pattern
|
|
40
|
-
|
|
41
|
-
:type config: dict
|
|
42
41
|
"""
|
|
43
42
|
|
|
44
43
|
NAME_PATTERN_CONSTRAINT = re.compile(r"\(\?P<tileid>\\d\{6\}\)")
|
|
@@ -60,9 +59,7 @@ class FilterLatestByName(Crunch):
|
|
|
60
59
|
"""Execute crunch: Filter Search results to get only the latest product, based on the name of the product
|
|
61
60
|
|
|
62
61
|
:param products: A list of products resulting from a search
|
|
63
|
-
:type products: list(:class:`~eodag.api.product._product.EOProduct`)
|
|
64
62
|
:returns: The filtered products
|
|
65
|
-
:rtype: list(:class:`~eodag.api.product._product.EOProduct`)
|
|
66
63
|
"""
|
|
67
64
|
logger.debug("Starting products filtering")
|
|
68
65
|
processed: List[str] = []
|
|
@@ -48,7 +48,6 @@ class FilterOverlap(Crunch):
|
|
|
48
48
|
- `within` : True if product geometry is within the search area
|
|
49
49
|
|
|
50
50
|
These configuration parameters are mutually exclusive.
|
|
51
|
-
:type config: dict
|
|
52
51
|
"""
|
|
53
52
|
|
|
54
53
|
def proceed(
|
|
@@ -57,11 +56,8 @@ class FilterOverlap(Crunch):
|
|
|
57
56
|
"""Execute crunch: Filter products, retaining only those that are overlapping with the search_extent
|
|
58
57
|
|
|
59
58
|
:param products: A list of products resulting from a search
|
|
60
|
-
:type products: list(:class:`~eodag.api.product._product.EOProduct`)
|
|
61
59
|
:param search_params: Search criteria that must contain `geometry`
|
|
62
|
-
:type search_params: dict
|
|
63
60
|
:returns: The filtered products
|
|
64
|
-
:rtype: list(:class:`~eodag.api.product._product.EOProduct`)
|
|
65
61
|
"""
|
|
66
62
|
logger.debug("Start filtering for overlapping products")
|
|
67
63
|
filtered: List[EOProduct] = []
|
|
@@ -73,10 +69,10 @@ class FilterOverlap(Crunch):
|
|
|
73
69
|
"geometry not found in cruncher arguments, filtering disabled."
|
|
74
70
|
)
|
|
75
71
|
return products
|
|
76
|
-
minimum_overlap = float(self.config.get("minimum_overlap", "0"))
|
|
77
|
-
contains = self.config.get("contains", False)
|
|
78
|
-
intersects = self.config.get("intersects", False)
|
|
79
|
-
within = self.config.get("within", False)
|
|
72
|
+
minimum_overlap = float(self.config.__dict__.get("minimum_overlap", "0"))
|
|
73
|
+
contains = self.config.__dict__.get("contains", False)
|
|
74
|
+
intersects = self.config.__dict__.get("intersects", False)
|
|
75
|
+
within = self.config.__dict__.get("within", False)
|
|
80
76
|
|
|
81
77
|
if contains and (within or intersects) or (within and intersects):
|
|
82
78
|
logger.warning(
|