eodag 3.0.0b2__py3-none-any.whl → 3.0.1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- eodag/__init__.py +6 -8
- eodag/api/core.py +295 -287
- eodag/api/product/__init__.py +10 -4
- eodag/api/product/_assets.py +2 -14
- eodag/api/product/_product.py +16 -30
- eodag/api/product/drivers/__init__.py +7 -2
- eodag/api/product/drivers/base.py +0 -3
- eodag/api/product/metadata_mapping.py +12 -31
- eodag/api/search_result.py +33 -12
- eodag/cli.py +35 -19
- eodag/config.py +455 -155
- eodag/plugins/apis/base.py +13 -7
- eodag/plugins/apis/ecmwf.py +16 -7
- eodag/plugins/apis/usgs.py +68 -16
- eodag/plugins/authentication/aws_auth.py +25 -7
- eodag/plugins/authentication/base.py +10 -1
- eodag/plugins/authentication/generic.py +14 -3
- eodag/plugins/authentication/header.py +12 -4
- eodag/plugins/authentication/keycloak.py +41 -22
- eodag/plugins/authentication/oauth.py +11 -1
- eodag/plugins/authentication/openid_connect.py +183 -167
- eodag/plugins/authentication/qsauth.py +12 -4
- eodag/plugins/authentication/sas_auth.py +19 -2
- eodag/plugins/authentication/token.py +59 -11
- eodag/plugins/authentication/token_exchange.py +19 -19
- eodag/plugins/crunch/base.py +7 -2
- eodag/plugins/crunch/filter_date.py +8 -11
- eodag/plugins/crunch/filter_latest_intersect.py +5 -7
- eodag/plugins/crunch/filter_latest_tpl_name.py +2 -5
- eodag/plugins/crunch/filter_overlap.py +9 -15
- eodag/plugins/crunch/filter_property.py +9 -14
- eodag/plugins/download/aws.py +84 -99
- eodag/plugins/download/base.py +36 -77
- eodag/plugins/download/creodias_s3.py +11 -2
- eodag/plugins/download/http.py +134 -109
- eodag/plugins/download/s3rest.py +37 -43
- eodag/plugins/manager.py +173 -41
- eodag/plugins/search/__init__.py +9 -9
- eodag/plugins/search/base.py +35 -35
- eodag/plugins/search/build_search_result.py +55 -64
- eodag/plugins/search/cop_marine.py +113 -32
- eodag/plugins/search/creodias_s3.py +20 -8
- eodag/plugins/search/csw.py +41 -1
- eodag/plugins/search/data_request_search.py +119 -14
- eodag/plugins/search/qssearch.py +619 -197
- eodag/plugins/search/static_stac_search.py +25 -23
- eodag/resources/ext_product_types.json +1 -1
- eodag/resources/product_types.yml +211 -56
- eodag/resources/providers.yml +1762 -1809
- eodag/resources/stac.yml +3 -163
- eodag/resources/user_conf_template.yml +134 -119
- eodag/rest/config.py +1 -2
- eodag/rest/constants.py +0 -1
- eodag/rest/core.py +70 -92
- eodag/rest/errors.py +181 -0
- eodag/rest/server.py +24 -330
- eodag/rest/stac.py +105 -630
- eodag/rest/types/eodag_search.py +17 -15
- eodag/rest/types/queryables.py +5 -14
- eodag/rest/types/stac_search.py +18 -13
- eodag/rest/utils/rfc3339.py +0 -1
- eodag/types/__init__.py +24 -6
- eodag/types/download_args.py +14 -5
- eodag/types/queryables.py +1 -2
- eodag/types/search_args.py +10 -11
- eodag/types/whoosh.py +0 -2
- eodag/utils/__init__.py +97 -136
- eodag/utils/constraints.py +0 -8
- eodag/utils/exceptions.py +23 -9
- eodag/utils/import_system.py +0 -4
- eodag/utils/logging.py +37 -80
- eodag/utils/notebook.py +4 -4
- eodag/utils/requests.py +13 -23
- eodag/utils/rest.py +0 -4
- eodag/utils/stac_reader.py +3 -15
- {eodag-3.0.0b2.dist-info → eodag-3.0.1.dist-info}/METADATA +41 -24
- eodag-3.0.1.dist-info/RECORD +109 -0
- {eodag-3.0.0b2.dist-info → eodag-3.0.1.dist-info}/WHEEL +1 -1
- {eodag-3.0.0b2.dist-info → eodag-3.0.1.dist-info}/entry_points.txt +1 -0
- eodag/resources/constraints/climate-dt.json +0 -13
- eodag/resources/constraints/extremes-dt.json +0 -8
- eodag-3.0.0b2.dist-info/RECORD +0 -110
- {eodag-3.0.0b2.dist-info → eodag-3.0.1.dist-info}/LICENSE +0 -0
- {eodag-3.0.0b2.dist-info → eodag-3.0.1.dist-info}/top_level.txt +0 -0
eodag/api/core.py
CHANGED
|
@@ -17,13 +17,25 @@
|
|
|
17
17
|
# limitations under the License.
|
|
18
18
|
from __future__ import annotations
|
|
19
19
|
|
|
20
|
+
import datetime
|
|
20
21
|
import logging
|
|
21
22
|
import os
|
|
22
23
|
import re
|
|
23
24
|
import shutil
|
|
24
25
|
import tempfile
|
|
25
26
|
from operator import itemgetter
|
|
26
|
-
from typing import
|
|
27
|
+
from typing import (
|
|
28
|
+
TYPE_CHECKING,
|
|
29
|
+
Annotated,
|
|
30
|
+
Any,
|
|
31
|
+
Dict,
|
|
32
|
+
Iterator,
|
|
33
|
+
List,
|
|
34
|
+
Optional,
|
|
35
|
+
Set,
|
|
36
|
+
Tuple,
|
|
37
|
+
Union,
|
|
38
|
+
)
|
|
27
39
|
|
|
28
40
|
import geojson
|
|
29
41
|
import pkg_resources
|
|
@@ -35,11 +47,16 @@ from whoosh.fields import Schema
|
|
|
35
47
|
from whoosh.index import create_in, exists_in, open_dir
|
|
36
48
|
from whoosh.qparser import QueryParser
|
|
37
49
|
|
|
38
|
-
from eodag.api.product.metadata_mapping import
|
|
50
|
+
from eodag.api.product.metadata_mapping import (
|
|
51
|
+
ONLINE_STATUS,
|
|
52
|
+
mtd_cfg_as_conversion_and_querypath,
|
|
53
|
+
)
|
|
39
54
|
from eodag.api.search_result import SearchResult
|
|
40
55
|
from eodag.config import (
|
|
56
|
+
PLUGINS_TOPICS_KEYS,
|
|
41
57
|
PluginConfig,
|
|
42
58
|
SimpleYamlProxyConfig,
|
|
59
|
+
credentials_in_auth,
|
|
43
60
|
get_ext_product_types_conf,
|
|
44
61
|
load_default_config,
|
|
45
62
|
load_stac_provider_config,
|
|
@@ -48,10 +65,12 @@ from eodag.config import (
|
|
|
48
65
|
override_config_from_file,
|
|
49
66
|
override_config_from_mapping,
|
|
50
67
|
provider_config_init,
|
|
68
|
+
share_credentials,
|
|
51
69
|
)
|
|
52
70
|
from eodag.plugins.manager import PluginManager
|
|
53
71
|
from eodag.plugins.search import PreparedSearch
|
|
54
72
|
from eodag.plugins.search.build_search_result import BuildPostSearchResult
|
|
73
|
+
from eodag.plugins.search.qssearch import PostJsonSearch
|
|
55
74
|
from eodag.types import model_fields_to_annotated
|
|
56
75
|
from eodag.types.queryables import CommonQueryables
|
|
57
76
|
from eodag.types.whoosh import EODAGQueryParser
|
|
@@ -66,17 +85,15 @@ from eodag.utils import (
|
|
|
66
85
|
MockResponse,
|
|
67
86
|
_deprecated,
|
|
68
87
|
copy_deepcopy,
|
|
69
|
-
deepcopy,
|
|
70
88
|
get_geometry_from_various,
|
|
71
89
|
makedirs,
|
|
72
90
|
obj_md5sum,
|
|
91
|
+
sort_dict,
|
|
73
92
|
string_to_jsonpath,
|
|
74
93
|
uri_to_path,
|
|
75
94
|
)
|
|
76
95
|
from eodag.utils.exceptions import (
|
|
77
|
-
AuthenticationError,
|
|
78
96
|
EodagError,
|
|
79
|
-
MisconfiguredError,
|
|
80
97
|
NoMatchingProductType,
|
|
81
98
|
PluginImplementationError,
|
|
82
99
|
RequestError,
|
|
@@ -96,7 +113,7 @@ if TYPE_CHECKING:
|
|
|
96
113
|
from eodag.plugins.search.base import Search
|
|
97
114
|
from eodag.types import ProviderSortables
|
|
98
115
|
from eodag.types.download_args import DownloadConf
|
|
99
|
-
from eodag.utils import
|
|
116
|
+
from eodag.utils import DownloadedCallback, ProgressCallback, Unpack
|
|
100
117
|
|
|
101
118
|
logger = logging.getLogger("eodag.core")
|
|
102
119
|
|
|
@@ -106,9 +123,7 @@ class EODataAccessGateway:
|
|
|
106
123
|
from different types of providers.
|
|
107
124
|
|
|
108
125
|
:param user_conf_file_path: (optional) Path to the user configuration file
|
|
109
|
-
:type user_conf_file_path: str
|
|
110
126
|
:param locations_conf_path: (optional) Path to the locations configuration file
|
|
111
|
-
:type locations_conf_path: str
|
|
112
127
|
"""
|
|
113
128
|
|
|
114
129
|
def __init__(
|
|
@@ -168,10 +183,15 @@ class EODataAccessGateway:
|
|
|
168
183
|
# Second level override: From environment variables
|
|
169
184
|
override_config_from_env(self.providers_config)
|
|
170
185
|
|
|
186
|
+
# share credentials between updated plugins confs
|
|
187
|
+
share_credentials(self.providers_config)
|
|
188
|
+
|
|
171
189
|
# init updated providers conf
|
|
172
|
-
stac_provider_config = load_stac_provider_config()
|
|
173
190
|
for provider in self.providers_config.keys():
|
|
174
|
-
provider_config_init(
|
|
191
|
+
provider_config_init(
|
|
192
|
+
self.providers_config[provider],
|
|
193
|
+
load_stac_provider_config(),
|
|
194
|
+
)
|
|
175
195
|
|
|
176
196
|
# re-build _plugins_manager using up-to-date providers_config
|
|
177
197
|
self._plugins_manager.rebuild(self.providers_config)
|
|
@@ -217,7 +237,6 @@ class EODataAccessGateway:
|
|
|
217
237
|
os.path.join(self.conf_dir, "shp"),
|
|
218
238
|
)
|
|
219
239
|
self.set_locations_conf(locations_conf_path)
|
|
220
|
-
self.search_errors: Set = set()
|
|
221
240
|
|
|
222
241
|
def get_version(self) -> str:
|
|
223
242
|
"""Get eodag package version"""
|
|
@@ -312,7 +331,6 @@ class EODataAccessGateway:
|
|
|
312
331
|
|
|
313
332
|
:param provider: The name of the provider that should be considered as the
|
|
314
333
|
preferred provider to be used for this instance
|
|
315
|
-
:type provider: str
|
|
316
334
|
"""
|
|
317
335
|
if provider not in self.available_providers():
|
|
318
336
|
raise UnsupportedProvider(
|
|
@@ -328,7 +346,6 @@ class EODataAccessGateway:
|
|
|
328
346
|
products, along with its priority.
|
|
329
347
|
|
|
330
348
|
:returns: The provider with the maximum priority and its priority
|
|
331
|
-
:rtype: tuple(str, int)
|
|
332
349
|
"""
|
|
333
350
|
providers_with_priority = [
|
|
334
351
|
(provider, conf.priority)
|
|
@@ -337,15 +354,24 @@ class EODataAccessGateway:
|
|
|
337
354
|
preferred, priority = max(providers_with_priority, key=itemgetter(1))
|
|
338
355
|
return preferred, priority
|
|
339
356
|
|
|
340
|
-
def update_providers_config(
|
|
357
|
+
def update_providers_config(
|
|
358
|
+
self,
|
|
359
|
+
yaml_conf: Optional[str] = None,
|
|
360
|
+
dict_conf: Optional[Dict[str, Any]] = None,
|
|
361
|
+
) -> None:
|
|
341
362
|
"""Update providers configuration with given input.
|
|
342
363
|
Can be used to add a provider to existing configuration or update
|
|
343
364
|
an existing one.
|
|
344
365
|
|
|
345
366
|
:param yaml_conf: YAML formated provider configuration
|
|
346
|
-
:
|
|
367
|
+
:param dict_conf: provider configuration as dictionary in place of ``yaml_conf``
|
|
347
368
|
"""
|
|
348
|
-
|
|
369
|
+
if dict_conf is not None:
|
|
370
|
+
conf_update = dict_conf
|
|
371
|
+
elif yaml_conf is not None:
|
|
372
|
+
conf_update = yaml.safe_load(yaml_conf)
|
|
373
|
+
else:
|
|
374
|
+
return None
|
|
349
375
|
|
|
350
376
|
# restore the pruned configuration
|
|
351
377
|
for provider in list(self._pruned_providers_config.keys()):
|
|
@@ -358,48 +384,86 @@ class EODataAccessGateway:
|
|
|
358
384
|
provider
|
|
359
385
|
)
|
|
360
386
|
|
|
361
|
-
# check if metada-mapping as already been built as jsonpath in providers_config
|
|
362
|
-
for provider, provider_conf in conf_update.items():
|
|
363
|
-
if (
|
|
364
|
-
provider in self.providers_config
|
|
365
|
-
and "metadata_mapping" in provider_conf.get("search", {})
|
|
366
|
-
):
|
|
367
|
-
search_plugin_key = "search"
|
|
368
|
-
elif (
|
|
369
|
-
provider in self.providers_config
|
|
370
|
-
and "metadata_mapping" in provider_conf.get("api", {})
|
|
371
|
-
):
|
|
372
|
-
search_plugin_key = "api"
|
|
373
|
-
else:
|
|
374
|
-
continue
|
|
375
|
-
# get some already configured value
|
|
376
|
-
configured_metadata_mapping = getattr(
|
|
377
|
-
self.providers_config[provider], search_plugin_key
|
|
378
|
-
).metadata_mapping
|
|
379
|
-
some_configured_value = next(iter(configured_metadata_mapping.values()))
|
|
380
|
-
# check if the configured value has already been built as jsonpath
|
|
381
|
-
if (
|
|
382
|
-
isinstance(some_configured_value, list)
|
|
383
|
-
and isinstance(some_configured_value[1], tuple)
|
|
384
|
-
or isinstance(some_configured_value, tuple)
|
|
385
|
-
):
|
|
386
|
-
# also build as jsonpath the incoming conf
|
|
387
|
-
mtd_cfg_as_conversion_and_querypath(
|
|
388
|
-
deepcopy(
|
|
389
|
-
conf_update[provider][search_plugin_key]["metadata_mapping"]
|
|
390
|
-
),
|
|
391
|
-
conf_update[provider][search_plugin_key]["metadata_mapping"],
|
|
392
|
-
)
|
|
393
|
-
|
|
394
387
|
override_config_from_mapping(self.providers_config, conf_update)
|
|
395
388
|
|
|
396
|
-
|
|
389
|
+
# share credentials between updated plugins confs
|
|
390
|
+
share_credentials(self.providers_config)
|
|
391
|
+
|
|
397
392
|
for provider in conf_update.keys():
|
|
398
|
-
provider_config_init(
|
|
393
|
+
provider_config_init(
|
|
394
|
+
self.providers_config[provider],
|
|
395
|
+
load_stac_provider_config(),
|
|
396
|
+
)
|
|
399
397
|
setattr(self.providers_config[provider], "product_types_fetched", False)
|
|
400
398
|
# re-create _plugins_manager using up-to-date providers_config
|
|
401
399
|
self._plugins_manager.build_product_type_to_provider_config_map()
|
|
402
400
|
|
|
401
|
+
def add_provider(
|
|
402
|
+
self,
|
|
403
|
+
name: str,
|
|
404
|
+
url: Optional[str] = None,
|
|
405
|
+
priority: Optional[int] = None,
|
|
406
|
+
search: Dict[str, Any] = {"type": "StacSearch"},
|
|
407
|
+
products: Dict[str, Any] = {
|
|
408
|
+
GENERIC_PRODUCT_TYPE: {"productType": "{productType}"}
|
|
409
|
+
},
|
|
410
|
+
download: Dict[str, Any] = {"type": "HTTPDownload", "auth_error_code": 401},
|
|
411
|
+
**kwargs: Dict[str, Any],
|
|
412
|
+
):
|
|
413
|
+
"""Adds a new provider.
|
|
414
|
+
|
|
415
|
+
``search``, ``products`` & ``download`` already have default values that will be
|
|
416
|
+
updated (not replaced), with user provided ones:
|
|
417
|
+
|
|
418
|
+
* ``search`` : ``{"type": "StacSearch"}``
|
|
419
|
+
* ``products`` : ``{"GENERIC_PRODUCT_TYPE": {"productType": "{productType}"}}``
|
|
420
|
+
* ``download`` : ``{"type": "HTTPDownload", "auth_error_code": 401}``
|
|
421
|
+
|
|
422
|
+
:param name: Name of provider
|
|
423
|
+
:param url: Provider url, also used as ``search["api_endpoint"]`` if not defined
|
|
424
|
+
:param priority: Provider priority. If None, provider will be set as preferred (highest priority)
|
|
425
|
+
:param search: Search :class:`~eodag.config.PluginConfig` mapping
|
|
426
|
+
:param products: Provider product types mapping
|
|
427
|
+
:param download: Download :class:`~eodag.config.PluginConfig` mapping
|
|
428
|
+
:param kwargs: Additional :class:`~eodag.config.ProviderConfig` mapping
|
|
429
|
+
"""
|
|
430
|
+
conf_dict: Dict[str, Any] = {
|
|
431
|
+
name: {
|
|
432
|
+
"url": url,
|
|
433
|
+
"search": {"type": "StacSearch", **search},
|
|
434
|
+
"products": {
|
|
435
|
+
GENERIC_PRODUCT_TYPE: {"productType": "{productType}"},
|
|
436
|
+
**products,
|
|
437
|
+
},
|
|
438
|
+
"download": {
|
|
439
|
+
"type": "HTTPDownload",
|
|
440
|
+
"auth_error_code": 401,
|
|
441
|
+
**download,
|
|
442
|
+
},
|
|
443
|
+
**kwargs,
|
|
444
|
+
}
|
|
445
|
+
}
|
|
446
|
+
if priority is not None:
|
|
447
|
+
conf_dict[name]["priority"] = priority
|
|
448
|
+
# if provided, use url as default search api_endpoint
|
|
449
|
+
if (
|
|
450
|
+
url
|
|
451
|
+
and conf_dict[name].get("search", {})
|
|
452
|
+
and not conf_dict[name]["search"].get("api_endpoint")
|
|
453
|
+
):
|
|
454
|
+
conf_dict[name]["search"]["api_endpoint"] = url
|
|
455
|
+
|
|
456
|
+
# api plugin usage: remove unneeded search/download/auth plugin conf
|
|
457
|
+
if conf_dict[name].get("api"):
|
|
458
|
+
for k in PLUGINS_TOPICS_KEYS:
|
|
459
|
+
if k != "api":
|
|
460
|
+
conf_dict[name].pop(k, None)
|
|
461
|
+
|
|
462
|
+
self.update_providers_config(dict_conf=conf_dict)
|
|
463
|
+
|
|
464
|
+
if priority is None:
|
|
465
|
+
self.set_preferred_provider(name)
|
|
466
|
+
|
|
403
467
|
def _prune_providers_list(self) -> None:
|
|
404
468
|
"""Removes from config providers needing auth that have no credentials set."""
|
|
405
469
|
update_needed = False
|
|
@@ -421,12 +485,7 @@ class EODataAccessGateway:
|
|
|
421
485
|
|
|
422
486
|
# check authentication
|
|
423
487
|
if hasattr(conf, "api") and getattr(conf.api, "need_auth", False):
|
|
424
|
-
credentials_exist =
|
|
425
|
-
[
|
|
426
|
-
cred is not None
|
|
427
|
-
for cred in getattr(conf.api, "credentials", {}).values()
|
|
428
|
-
]
|
|
429
|
-
)
|
|
488
|
+
credentials_exist = credentials_in_auth(conf.api)
|
|
430
489
|
if not credentials_exist:
|
|
431
490
|
# credentials needed but not found
|
|
432
491
|
self._pruned_providers_config[provider] = self.providers_config.pop(
|
|
@@ -438,7 +497,7 @@ class EODataAccessGateway:
|
|
|
438
497
|
provider,
|
|
439
498
|
)
|
|
440
499
|
elif hasattr(conf, "search") and getattr(conf.search, "need_auth", False):
|
|
441
|
-
if not hasattr(conf, "auth"):
|
|
500
|
+
if not hasattr(conf, "auth") and not hasattr(conf, "search_auth"):
|
|
442
501
|
# credentials needed but no auth plugin was found
|
|
443
502
|
self._pruned_providers_config[provider] = self.providers_config.pop(
|
|
444
503
|
provider
|
|
@@ -449,11 +508,13 @@ class EODataAccessGateway:
|
|
|
449
508
|
provider,
|
|
450
509
|
)
|
|
451
510
|
continue
|
|
452
|
-
credentials_exist =
|
|
453
|
-
|
|
454
|
-
|
|
455
|
-
|
|
456
|
-
|
|
511
|
+
credentials_exist = (
|
|
512
|
+
hasattr(conf, "search_auth")
|
|
513
|
+
and credentials_in_auth(conf.search_auth)
|
|
514
|
+
) or (
|
|
515
|
+
not hasattr(conf, "search_auth")
|
|
516
|
+
and hasattr(conf, "auth")
|
|
517
|
+
and credentials_in_auth(conf.auth)
|
|
457
518
|
)
|
|
458
519
|
if not credentials_exist:
|
|
459
520
|
# credentials needed but not found
|
|
@@ -502,7 +563,6 @@ class EODataAccessGateway:
|
|
|
502
563
|
attr: FRA
|
|
503
564
|
|
|
504
565
|
:param locations_conf_path: Path to the locations configuration file
|
|
505
|
-
:type locations_conf_path: str
|
|
506
566
|
"""
|
|
507
567
|
if os.path.isfile(locations_conf_path):
|
|
508
568
|
locations_config = load_yml_config(locations_conf_path)
|
|
@@ -525,12 +585,9 @@ class EODataAccessGateway:
|
|
|
525
585
|
|
|
526
586
|
:param provider: (optional) The name of a provider that must support the product
|
|
527
587
|
types we are about to list
|
|
528
|
-
:type provider: str
|
|
529
588
|
:param fetch_providers: (optional) Whether to fetch providers for new product
|
|
530
589
|
types or not
|
|
531
|
-
:type fetch_providers: bool
|
|
532
590
|
:returns: The list of the product types that can be accessed using eodag.
|
|
533
|
-
:rtype: list(dict)
|
|
534
591
|
:raises: :class:`~eodag.utils.exceptions.UnsupportedProvider`
|
|
535
592
|
"""
|
|
536
593
|
if fetch_providers:
|
|
@@ -572,22 +629,32 @@ class EODataAccessGateway:
|
|
|
572
629
|
def fetch_product_types_list(self, provider: Optional[str] = None) -> None:
|
|
573
630
|
"""Fetch product types list and update if needed
|
|
574
631
|
|
|
575
|
-
:param provider:
|
|
576
|
-
should be updated. Defaults to all providers (None value).
|
|
577
|
-
:type provider: str
|
|
632
|
+
:param provider: The name of a provider or provider-group for which product types
|
|
633
|
+
list should be updated. Defaults to all providers (None value).
|
|
578
634
|
"""
|
|
635
|
+
providers_to_fetch = list(self.providers_config.keys())
|
|
636
|
+
# check if some providers are grouped under a group name which is not a provider name
|
|
579
637
|
if provider is not None and provider not in self.providers_config:
|
|
580
|
-
|
|
638
|
+
providers_to_fetch = [
|
|
639
|
+
p
|
|
640
|
+
for p, pconf in self.providers_config.items()
|
|
641
|
+
if provider == getattr(pconf, "group", None)
|
|
642
|
+
]
|
|
643
|
+
if providers_to_fetch:
|
|
644
|
+
logger.info(
|
|
645
|
+
f"Fetch product types for {provider} group: {', '.join(providers_to_fetch)}"
|
|
646
|
+
)
|
|
647
|
+
else:
|
|
648
|
+
return None
|
|
649
|
+
elif provider is not None:
|
|
650
|
+
providers_to_fetch = [provider]
|
|
581
651
|
|
|
582
652
|
# providers discovery confs that are fetchable
|
|
583
653
|
providers_discovery_configs_fetchable: Dict[str, Any] = {}
|
|
584
654
|
# check if any provider has not already been fetched for product types
|
|
585
655
|
already_fetched = True
|
|
586
|
-
for provider_to_fetch
|
|
587
|
-
|
|
588
|
-
if provider
|
|
589
|
-
else self.providers_config.items()
|
|
590
|
-
):
|
|
656
|
+
for provider_to_fetch in providers_to_fetch:
|
|
657
|
+
provider_config = self.providers_config[provider_to_fetch]
|
|
591
658
|
# get discovery conf
|
|
592
659
|
if hasattr(provider_config, "search"):
|
|
593
660
|
provider_search_config = provider_config.search
|
|
@@ -709,13 +776,20 @@ class EODataAccessGateway:
|
|
|
709
776
|
) -> Optional[Dict[str, Any]]:
|
|
710
777
|
"""Fetch providers for product types
|
|
711
778
|
|
|
712
|
-
:param provider:
|
|
713
|
-
providers (None value).
|
|
714
|
-
:type provider: str
|
|
779
|
+
:param provider: The name of a provider or provider-group to fetch. Defaults to
|
|
780
|
+
all providers (None value).
|
|
715
781
|
:returns: external product types configuration
|
|
716
|
-
:rtype: dict
|
|
717
782
|
"""
|
|
718
|
-
|
|
783
|
+
grouped_providers = [
|
|
784
|
+
p
|
|
785
|
+
for p, provider_config in self.providers_config.items()
|
|
786
|
+
if provider == getattr(provider_config, "group", None)
|
|
787
|
+
]
|
|
788
|
+
if provider and provider not in self.providers_config and grouped_providers:
|
|
789
|
+
logger.info(
|
|
790
|
+
f"Discover product types for {provider} group: {', '.join(grouped_providers)}"
|
|
791
|
+
)
|
|
792
|
+
elif provider and provider not in self.providers_config:
|
|
719
793
|
raise UnsupportedProvider(
|
|
720
794
|
f"The requested provider is not (yet) supported: {provider}"
|
|
721
795
|
)
|
|
@@ -724,7 +798,9 @@ class EODataAccessGateway:
|
|
|
724
798
|
p
|
|
725
799
|
for p in (
|
|
726
800
|
[
|
|
727
|
-
|
|
801
|
+
p
|
|
802
|
+
for p in self.providers_config
|
|
803
|
+
if p in grouped_providers + [provider]
|
|
728
804
|
]
|
|
729
805
|
if provider
|
|
730
806
|
else self.available_providers()
|
|
@@ -738,29 +814,28 @@ class EODataAccessGateway:
|
|
|
738
814
|
search_plugin_config = self.providers_config[provider].api
|
|
739
815
|
else:
|
|
740
816
|
return None
|
|
741
|
-
if getattr(search_plugin_config, "discover_product_types",
|
|
817
|
+
if getattr(search_plugin_config, "discover_product_types", {}).get(
|
|
818
|
+
"fetch_url", None
|
|
819
|
+
):
|
|
742
820
|
search_plugin: Union[Search, Api] = next(
|
|
743
821
|
self._plugins_manager.get_search_plugins(provider=provider)
|
|
744
822
|
)
|
|
823
|
+
# check after plugin init if still fetchable
|
|
824
|
+
if not getattr(search_plugin.config, "discover_product_types", {}).get(
|
|
825
|
+
"fetch_url"
|
|
826
|
+
):
|
|
827
|
+
continue
|
|
745
828
|
# append auth to search plugin if needed
|
|
746
829
|
if getattr(search_plugin.config, "need_auth", False):
|
|
747
|
-
|
|
748
|
-
search_plugin.provider
|
|
749
|
-
|
|
750
|
-
|
|
751
|
-
getattr(auth_plugin, "authenticate", None)
|
|
830
|
+
if auth := self._plugins_manager.get_auth(
|
|
831
|
+
search_plugin.provider,
|
|
832
|
+
getattr(search_plugin.config, "api_endpoint", None),
|
|
833
|
+
search_plugin.config,
|
|
752
834
|
):
|
|
753
|
-
|
|
754
|
-
kwargs["auth"] = auth_plugin.authenticate()
|
|
755
|
-
except (AuthenticationError, MisconfiguredError) as e:
|
|
756
|
-
logger.warning(
|
|
757
|
-
f"Could not authenticate on {provider}: {str(e)}"
|
|
758
|
-
)
|
|
759
|
-
ext_product_types_conf[provider] = None
|
|
760
|
-
continue
|
|
835
|
+
kwargs["auth"] = auth
|
|
761
836
|
else:
|
|
762
|
-
logger.
|
|
763
|
-
f"Could not authenticate on {provider}
|
|
837
|
+
logger.debug(
|
|
838
|
+
f"Could not authenticate on {provider} for product types discovery"
|
|
764
839
|
)
|
|
765
840
|
ext_product_types_conf[provider] = None
|
|
766
841
|
continue
|
|
@@ -769,7 +844,7 @@ class EODataAccessGateway:
|
|
|
769
844
|
**kwargs
|
|
770
845
|
)
|
|
771
846
|
|
|
772
|
-
return ext_product_types_conf
|
|
847
|
+
return sort_dict(ext_product_types_conf)
|
|
773
848
|
|
|
774
849
|
def update_product_types_list(
|
|
775
850
|
self, ext_product_types_conf: Dict[str, Optional[Dict[str, Dict[str, Any]]]]
|
|
@@ -777,7 +852,6 @@ class EODataAccessGateway:
|
|
|
777
852
|
"""Update eodag product types list
|
|
778
853
|
|
|
779
854
|
:param ext_product_types_conf: external product types configuration
|
|
780
|
-
:type ext_product_types_conf: dict
|
|
781
855
|
"""
|
|
782
856
|
for provider, new_product_types_conf in ext_product_types_conf.items():
|
|
783
857
|
if new_product_types_conf and provider in self.providers_config:
|
|
@@ -787,7 +861,9 @@ class EODataAccessGateway:
|
|
|
787
861
|
) or getattr(self.providers_config[provider], "api", None)
|
|
788
862
|
if search_plugin_config is None:
|
|
789
863
|
continue
|
|
790
|
-
if not
|
|
864
|
+
if not getattr(
|
|
865
|
+
search_plugin_config, "discover_product_types", {}
|
|
866
|
+
).get("fetch_url", None):
|
|
791
867
|
# conf has been updated and provider product types are no more discoverable
|
|
792
868
|
continue
|
|
793
869
|
provider_products_config = (
|
|
@@ -846,7 +922,7 @@ class EODataAccessGateway:
|
|
|
846
922
|
new_product_types.append(new_product_type)
|
|
847
923
|
if new_product_types:
|
|
848
924
|
logger.debug(
|
|
849
|
-
f"Added
|
|
925
|
+
f"Added {len(new_product_types)} product types for {provider}"
|
|
850
926
|
)
|
|
851
927
|
|
|
852
928
|
elif provider not in self.providers_config:
|
|
@@ -870,12 +946,9 @@ class EODataAccessGateway:
|
|
|
870
946
|
priority level.
|
|
871
947
|
|
|
872
948
|
:param product_type: (optional) Only list providers configured for this product_type
|
|
873
|
-
:type product_type: Optional[str]
|
|
874
949
|
:param by_group: (optional) If set to True, list groups when available instead
|
|
875
950
|
of providers, mixed with other providers
|
|
876
|
-
:type by_group: bool
|
|
877
951
|
:returns: the sorted list of the available providers or groups
|
|
878
|
-
:rtype: List[str]
|
|
879
952
|
"""
|
|
880
953
|
|
|
881
954
|
if product_type:
|
|
@@ -909,9 +982,7 @@ class EODataAccessGateway:
|
|
|
909
982
|
|
|
910
983
|
:param alias_or_id: Alias of the product type. If an existing ID is given, this
|
|
911
984
|
method will directly return the given value.
|
|
912
|
-
:type alias_or_id: str
|
|
913
985
|
:returns: Internal name of the product type.
|
|
914
|
-
:rtype: str
|
|
915
986
|
"""
|
|
916
987
|
product_types = [
|
|
917
988
|
k
|
|
@@ -939,9 +1010,7 @@ class EODataAccessGateway:
|
|
|
939
1010
|
given product type, its ID is returned instead.
|
|
940
1011
|
|
|
941
1012
|
:param product_type: product type ID
|
|
942
|
-
:type product_type: str
|
|
943
1013
|
:returns: Alias of the product type or its ID if no alias has been defined for it.
|
|
944
|
-
:rtype: str
|
|
945
1014
|
"""
|
|
946
1015
|
if product_type not in self.product_types_config:
|
|
947
1016
|
raise NoMatchingProductType(product_type)
|
|
@@ -972,31 +1041,18 @@ class EODataAccessGateway:
|
|
|
972
1041
|
|
|
973
1042
|
:param free_text: Whoosh-compatible free text search filter used to search
|
|
974
1043
|
accross all the following parameters
|
|
975
|
-
:type free_text: Optional[str]
|
|
976
1044
|
:param intersect: Join results for each parameter using INTERSECT instead of UNION.
|
|
977
|
-
:type intersect: bool
|
|
978
1045
|
:param instrument: Instrument parameter.
|
|
979
|
-
:type instrument: Optional[str]
|
|
980
1046
|
:param platform: Platform parameter.
|
|
981
|
-
:type platform: Optional[str]
|
|
982
1047
|
:param platformSerialIdentifier: Platform serial identifier parameter.
|
|
983
|
-
:type platformSerialIdentifier: Optional[str]
|
|
984
1048
|
:param processingLevel: Processing level parameter.
|
|
985
|
-
:type processingLevel: Optional[str]
|
|
986
1049
|
:param sensorType: Sensor type parameter.
|
|
987
|
-
:type sensorType: Optional[str]
|
|
988
1050
|
:param keywords: Keywords parameter.
|
|
989
|
-
:type keywords: Optional[str]
|
|
990
1051
|
:param abstract: Abstract parameter.
|
|
991
|
-
:type abstract: Optional[str]
|
|
992
1052
|
:param title: Title parameter.
|
|
993
|
-
:type title: Optional[str]
|
|
994
1053
|
:param missionStartDate: start date for datetime filtering. Not used by free_text
|
|
995
|
-
:type missionStartDate: Optional[str]
|
|
996
1054
|
:param missionEndDate: end date for datetime filtering. Not used by free_text
|
|
997
|
-
:type missionEndDate: Optional[str]
|
|
998
1055
|
:returns: The best match for the given parameters.
|
|
999
|
-
:rtype: List[str]
|
|
1000
1056
|
:raises: :class:`~eodag.utils.exceptions.NoMatchingProductType`
|
|
1001
1057
|
"""
|
|
1002
1058
|
if productType := kwargs.get("productType"):
|
|
@@ -1038,20 +1094,28 @@ class EODataAccessGateway:
|
|
|
1038
1094
|
|
|
1039
1095
|
# datetime filtering
|
|
1040
1096
|
if missionStartDate or missionEndDate:
|
|
1097
|
+
min_aware = datetime.datetime.min.replace(tzinfo=datetime.timezone.utc)
|
|
1098
|
+
max_aware = datetime.datetime.max.replace(tzinfo=datetime.timezone.utc)
|
|
1041
1099
|
guesses = [
|
|
1042
1100
|
g
|
|
1043
1101
|
for g in guesses
|
|
1044
1102
|
if (
|
|
1045
|
-
|
|
1046
|
-
|
|
1047
|
-
|
|
1048
|
-
|
|
1049
|
-
|
|
1050
|
-
|
|
1051
|
-
|
|
1052
|
-
|
|
1053
|
-
|
|
1054
|
-
|
|
1103
|
+
max(
|
|
1104
|
+
rfc3339_str_to_datetime(missionStartDate)
|
|
1105
|
+
if missionStartDate
|
|
1106
|
+
else min_aware,
|
|
1107
|
+
rfc3339_str_to_datetime(g["missionStartDate"])
|
|
1108
|
+
if g.get("missionStartDate")
|
|
1109
|
+
else min_aware,
|
|
1110
|
+
)
|
|
1111
|
+
<= min(
|
|
1112
|
+
rfc3339_str_to_datetime(missionEndDate)
|
|
1113
|
+
if missionEndDate
|
|
1114
|
+
else max_aware,
|
|
1115
|
+
rfc3339_str_to_datetime(g["missionEndDate"])
|
|
1116
|
+
if g.get("missionEndDate")
|
|
1117
|
+
else max_aware,
|
|
1118
|
+
)
|
|
1055
1119
|
)
|
|
1056
1120
|
]
|
|
1057
1121
|
|
|
@@ -1083,21 +1147,16 @@ class EODataAccessGateway:
|
|
|
1083
1147
|
Only if the request fails for all available providers, an error will be thrown.
|
|
1084
1148
|
|
|
1085
1149
|
:param page: (optional) The page number to return
|
|
1086
|
-
:type page: int
|
|
1087
1150
|
:param items_per_page: (optional) The number of results that must appear in one single
|
|
1088
1151
|
page
|
|
1089
|
-
:type items_per_page: int
|
|
1090
1152
|
:param raise_errors: (optional) When an error occurs when searching, if this is set to
|
|
1091
1153
|
True, the error is raised
|
|
1092
|
-
:type raise_errors: bool
|
|
1093
1154
|
:param start: (optional) Start sensing time in ISO 8601 format (e.g. "1990-11-26",
|
|
1094
1155
|
"1990-11-26T14:30:10.153Z", "1990-11-26T14:30:10+02:00", ...).
|
|
1095
1156
|
If no time offset is given, the time is assumed to be given in UTC.
|
|
1096
|
-
:type start: str
|
|
1097
1157
|
:param end: (optional) End sensing time in ISO 8601 format (e.g. "1990-11-26",
|
|
1098
1158
|
"1990-11-26T14:30:10.153Z", "1990-11-26T14:30:10+02:00", ...).
|
|
1099
1159
|
If no time offset is given, the time is assumed to be given in UTC.
|
|
1100
|
-
:type end: str
|
|
1101
1160
|
:param geom: (optional) Search area that can be defined in different ways:
|
|
1102
1161
|
|
|
1103
1162
|
* with a Shapely geometry object:
|
|
@@ -1107,24 +1166,22 @@ class EODataAccessGateway:
|
|
|
1107
1166
|
* with a bounding box as list of float:
|
|
1108
1167
|
``[lonmin, latmin, lonmax, latmax]``
|
|
1109
1168
|
* with a WKT str
|
|
1110
|
-
:type geom: Union[str, dict, shapely.geometry.base.BaseGeometry]
|
|
1111
1169
|
:param locations: (optional) Location filtering by name using locations configuration
|
|
1112
1170
|
``{"<location_name>"="<attr_regex>"}``. For example, ``{"country"="PA."}`` will use
|
|
1113
1171
|
the geometry of the features having the property ISO3 starting with
|
|
1114
1172
|
'PA' such as Panama and Pakistan in the shapefile configured with
|
|
1115
1173
|
name=country and attr=ISO3
|
|
1116
|
-
:type locations: dict
|
|
1117
1174
|
:param provider: (optional) the provider to be used. If set, search fallback will be disabled.
|
|
1118
1175
|
If not set, the configured preferred provider will be used at first
|
|
1119
1176
|
before trying others until finding results.
|
|
1120
|
-
:type provider: str
|
|
1121
1177
|
:param count: (optional) Whether to run a query with a count request or not
|
|
1122
|
-
:type count: bool
|
|
1123
1178
|
:param kwargs: Some other criteria that will be used to do the search,
|
|
1124
1179
|
using paramaters compatibles with the provider
|
|
1125
|
-
:type kwargs: Union[int, str, bool, dict]
|
|
1126
1180
|
:returns: A collection of EO products matching the criteria
|
|
1127
|
-
|
|
1181
|
+
|
|
1182
|
+
.. versionchanged:: v3.0.0b1
|
|
1183
|
+
``search()`` method now returns only a single :class:`~eodag.api.search_result.SearchResult`
|
|
1184
|
+
instead of a 2 values tuple.
|
|
1128
1185
|
|
|
1129
1186
|
.. note::
|
|
1130
1187
|
The search interfaces, which are implemented as plugins, are required to
|
|
@@ -1154,7 +1211,7 @@ class EODataAccessGateway:
|
|
|
1154
1211
|
items_per_page=items_per_page,
|
|
1155
1212
|
)
|
|
1156
1213
|
|
|
1157
|
-
|
|
1214
|
+
errors: List[Tuple[str, Exception]] = []
|
|
1158
1215
|
# Loop over available providers and return the first non-empty results
|
|
1159
1216
|
for i, search_plugin in enumerate(search_plugins):
|
|
1160
1217
|
search_plugin.clear()
|
|
@@ -1164,17 +1221,19 @@ class EODataAccessGateway:
|
|
|
1164
1221
|
raise_errors=raise_errors,
|
|
1165
1222
|
**search_kwargs,
|
|
1166
1223
|
)
|
|
1224
|
+
errors.extend(search_results.errors)
|
|
1167
1225
|
if len(search_results) == 0 and i < len(search_plugins) - 1:
|
|
1168
1226
|
logger.warning(
|
|
1169
1227
|
f"No result could be obtained from provider {search_plugin.provider}, "
|
|
1170
1228
|
"we will try to get the data from another provider",
|
|
1171
1229
|
)
|
|
1172
1230
|
elif len(search_results) > 0:
|
|
1231
|
+
search_results.errors = errors
|
|
1173
1232
|
return search_results
|
|
1174
1233
|
|
|
1175
1234
|
if i > 1:
|
|
1176
1235
|
logger.error("No result could be obtained from any available provider")
|
|
1177
|
-
return SearchResult([], 0) if count else SearchResult([])
|
|
1236
|
+
return SearchResult([], 0, errors) if count else SearchResult([], errors=errors)
|
|
1178
1237
|
|
|
1179
1238
|
def search_iter_page(
|
|
1180
1239
|
self,
|
|
@@ -1188,15 +1247,12 @@ class EODataAccessGateway:
|
|
|
1188
1247
|
"""Iterate over the pages of a products search.
|
|
1189
1248
|
|
|
1190
1249
|
:param items_per_page: (optional) The number of results requested per page
|
|
1191
|
-
:type items_per_page: int
|
|
1192
1250
|
:param start: (optional) Start sensing time in ISO 8601 format (e.g. "1990-11-26",
|
|
1193
1251
|
"1990-11-26T14:30:10.153Z", "1990-11-26T14:30:10+02:00", ...).
|
|
1194
1252
|
If no time offset is given, the time is assumed to be given in UTC.
|
|
1195
|
-
:type start: str
|
|
1196
1253
|
:param end: (optional) End sensing time in ISO 8601 format (e.g. "1990-11-26",
|
|
1197
1254
|
"1990-11-26T14:30:10.153Z", "1990-11-26T14:30:10+02:00", ...).
|
|
1198
1255
|
If no time offset is given, the time is assumed to be given in UTC.
|
|
1199
|
-
:type end: str
|
|
1200
1256
|
:param geom: (optional) Search area that can be defined in different ways:
|
|
1201
1257
|
|
|
1202
1258
|
* with a Shapely geometry object:
|
|
@@ -1206,19 +1262,15 @@ class EODataAccessGateway:
|
|
|
1206
1262
|
* with a bounding box as list of float:
|
|
1207
1263
|
``[lonmin, latmin, lonmax, latmax]``
|
|
1208
1264
|
* with a WKT str
|
|
1209
|
-
:type geom: Union[str, dict, shapely.geometry.base.BaseGeometry]
|
|
1210
1265
|
:param locations: (optional) Location filtering by name using locations configuration
|
|
1211
1266
|
``{"<location_name>"="<attr_regex>"}``. For example, ``{"country"="PA."}`` will use
|
|
1212
1267
|
the geometry of the features having the property ISO3 starting with
|
|
1213
1268
|
'PA' such as Panama and Pakistan in the shapefile configured with
|
|
1214
1269
|
name=country and attr=ISO3
|
|
1215
|
-
:type locations: dict
|
|
1216
1270
|
:param kwargs: Some other criteria that will be used to do the search,
|
|
1217
1271
|
using paramaters compatibles with the provider
|
|
1218
|
-
:type kwargs: Union[int, str, bool, dict]
|
|
1219
1272
|
:returns: An iterator that yields page per page a collection of EO products
|
|
1220
1273
|
matching the criteria
|
|
1221
|
-
:rtype: Iterator[:class:`~eodag.api.search_result.SearchResult`]
|
|
1222
1274
|
"""
|
|
1223
1275
|
search_plugins, search_kwargs = self._prepare_search(
|
|
1224
1276
|
start=start, end=end, geom=geom, locations=locations, **kwargs
|
|
@@ -1253,15 +1305,11 @@ class EODataAccessGateway:
|
|
|
1253
1305
|
"""Iterate over the pages of a products search using a given search plugin.
|
|
1254
1306
|
|
|
1255
1307
|
:param items_per_page: (optional) The number of results requested per page
|
|
1256
|
-
:type items_per_page: int
|
|
1257
1308
|
:param kwargs: Some other criteria that will be used to do the search,
|
|
1258
1309
|
using parameters compatibles with the provider
|
|
1259
|
-
:type kwargs: Union[int, str, bool, dict]
|
|
1260
1310
|
:param search_plugin: search plugin to be used
|
|
1261
|
-
:type search_plugin: eodag.plugins.search.base.Search
|
|
1262
1311
|
:returns: An iterator that yields page per page a collection of EO products
|
|
1263
1312
|
matching the criteria
|
|
1264
|
-
:rtype: Iterator[:class:`~eodag.api.search_result.SearchResult`]
|
|
1265
1313
|
"""
|
|
1266
1314
|
|
|
1267
1315
|
iteration = 1
|
|
@@ -1390,15 +1438,12 @@ class EODataAccessGateway:
|
|
|
1390
1438
|
matching the search criteria. If this number is not
|
|
1391
1439
|
available, a default value of 50 is used instead.
|
|
1392
1440
|
items_per_page can also be set to any arbitrary value.
|
|
1393
|
-
:type items_per_page: int
|
|
1394
1441
|
:param start: (optional) Start sensing time in ISO 8601 format (e.g. "1990-11-26",
|
|
1395
1442
|
"1990-11-26T14:30:10.153Z", "1990-11-26T14:30:10+02:00", ...).
|
|
1396
1443
|
If no time offset is given, the time is assumed to be given in UTC.
|
|
1397
|
-
:type start: str
|
|
1398
1444
|
:param end: (optional) End sensing time in ISO 8601 format (e.g. "1990-11-26",
|
|
1399
1445
|
"1990-11-26T14:30:10.153Z", "1990-11-26T14:30:10+02:00", ...).
|
|
1400
1446
|
If no time offset is given, the time is assumed to be given in UTC.
|
|
1401
|
-
:type end: str
|
|
1402
1447
|
:param geom: (optional) Search area that can be defined in different ways:
|
|
1403
1448
|
|
|
1404
1449
|
* with a Shapely geometry object:
|
|
@@ -1408,19 +1453,15 @@ class EODataAccessGateway:
|
|
|
1408
1453
|
* with a bounding box as list of float:
|
|
1409
1454
|
``[lonmin, latmin, lonmax, latmax]``
|
|
1410
1455
|
* with a WKT str
|
|
1411
|
-
:type geom: Union[str, dict, shapely.geometry.base.BaseGeometry]
|
|
1412
1456
|
:param locations: (optional) Location filtering by name using locations configuration
|
|
1413
1457
|
``{"<location_name>"="<attr_regex>"}``. For example, ``{"country"="PA."}`` will use
|
|
1414
1458
|
the geometry of the features having the property ISO3 starting with
|
|
1415
1459
|
'PA' such as Panama and Pakistan in the shapefile configured with
|
|
1416
1460
|
name=country and attr=ISO3
|
|
1417
|
-
:type locations: dict
|
|
1418
1461
|
:param kwargs: Some other criteria that will be used to do the search,
|
|
1419
1462
|
using parameters compatible with the provider
|
|
1420
|
-
:type kwargs: Union[int, str, bool, dict]
|
|
1421
1463
|
:returns: An iterator that yields page per page a collection of EO products
|
|
1422
1464
|
matching the criteria
|
|
1423
|
-
:rtype: Iterator[:class:`~eodag.api.search_result.SearchResult`]
|
|
1424
1465
|
"""
|
|
1425
1466
|
# Get the search plugin and the maximized value
|
|
1426
1467
|
# of items_per_page if defined for the provider used.
|
|
@@ -1452,7 +1493,7 @@ class EODataAccessGateway:
|
|
|
1452
1493
|
)
|
|
1453
1494
|
or DEFAULT_MAX_ITEMS_PER_PAGE
|
|
1454
1495
|
)
|
|
1455
|
-
logger.
|
|
1496
|
+
logger.info(
|
|
1456
1497
|
"Searching for all the products with provider %s and a maximum of %s "
|
|
1457
1498
|
"items per page.",
|
|
1458
1499
|
search_plugin.provider,
|
|
@@ -1509,22 +1550,18 @@ class EODataAccessGateway:
|
|
|
1509
1550
|
perform the search, if this information is available
|
|
1510
1551
|
|
|
1511
1552
|
:param uid: The uid of the EO product
|
|
1512
|
-
:type uid: str
|
|
1513
1553
|
:param provider: (optional) The provider on which to search the product.
|
|
1514
1554
|
This may be useful for performance reasons when the user
|
|
1515
1555
|
knows this product is available on the given provider
|
|
1516
|
-
:type provider: str
|
|
1517
1556
|
:param kwargs: Search criteria to help finding the right product
|
|
1518
|
-
:type kwargs: Any
|
|
1519
1557
|
:returns: A search result with one EO product or None at all
|
|
1520
|
-
:rtype: :class:`~eodag.api.search_result.SearchResult`
|
|
1521
1558
|
"""
|
|
1522
1559
|
product_type = kwargs.get("productType", None)
|
|
1523
1560
|
if product_type is not None:
|
|
1524
1561
|
try:
|
|
1525
1562
|
product_type = self.get_product_type_from_alias(product_type)
|
|
1526
1563
|
except NoMatchingProductType:
|
|
1527
|
-
logger.
|
|
1564
|
+
logger.debug("product type %s not found", product_type)
|
|
1528
1565
|
get_search_plugins_kwargs = dict(provider=provider, product_type=product_type)
|
|
1529
1566
|
search_plugins = self._plugins_manager.get_search_plugins(
|
|
1530
1567
|
**get_search_plugins_kwargs
|
|
@@ -1547,7 +1584,7 @@ class EODataAccessGateway:
|
|
|
1547
1584
|
"max_items_per_page", DEFAULT_MAX_ITEMS_PER_PAGE
|
|
1548
1585
|
)
|
|
1549
1586
|
kwargs.update(items_per_page=items_per_page)
|
|
1550
|
-
if isinstance(plugin,
|
|
1587
|
+
if isinstance(plugin, PostJsonSearch):
|
|
1551
1588
|
kwargs.update(
|
|
1552
1589
|
items_per_page=items_per_page,
|
|
1553
1590
|
_dc_qs=_dc_qs,
|
|
@@ -1565,9 +1602,10 @@ class EODataAccessGateway:
|
|
|
1565
1602
|
**kwargs,
|
|
1566
1603
|
):
|
|
1567
1604
|
results.data.extend(page_results.data)
|
|
1568
|
-
except Exception:
|
|
1605
|
+
except Exception as e:
|
|
1569
1606
|
if kwargs.get("raise_errors"):
|
|
1570
1607
|
raise
|
|
1608
|
+
logger.warning(e)
|
|
1571
1609
|
continue
|
|
1572
1610
|
|
|
1573
1611
|
# try using crunch to get unique result
|
|
@@ -1597,20 +1635,20 @@ class EODataAccessGateway:
|
|
|
1597
1635
|
plugins = self._plugins_manager.get_search_plugins(provider=provider)
|
|
1598
1636
|
plugin = next(plugins)
|
|
1599
1637
|
|
|
1638
|
+
# check after plugin init if still fetchable
|
|
1639
|
+
if not getattr(plugin.config, "discover_product_types", {}).get("fetch_url"):
|
|
1640
|
+
return None
|
|
1641
|
+
|
|
1600
1642
|
kwargs: Dict[str, Any] = {"productType": product_type}
|
|
1601
1643
|
|
|
1602
1644
|
# append auth if needed
|
|
1603
1645
|
if getattr(plugin.config, "need_auth", False):
|
|
1604
|
-
|
|
1605
|
-
|
|
1606
|
-
|
|
1607
|
-
|
|
1608
|
-
|
|
1609
|
-
|
|
1610
|
-
else:
|
|
1611
|
-
logger.warning(
|
|
1612
|
-
f"Could not authenticate on {provider} using {auth_plugin} plugin"
|
|
1613
|
-
)
|
|
1646
|
+
if auth := self._plugins_manager.get_auth(
|
|
1647
|
+
plugin.provider,
|
|
1648
|
+
getattr(plugin.config, "api_endpoint", None),
|
|
1649
|
+
plugin.config,
|
|
1650
|
+
):
|
|
1651
|
+
kwargs["auth"] = auth
|
|
1614
1652
|
|
|
1615
1653
|
product_type_config = plugin.discover_product_types(**kwargs)
|
|
1616
1654
|
self.update_product_types_list({provider: product_type_config})
|
|
@@ -1641,25 +1679,18 @@ class EODataAccessGateway:
|
|
|
1641
1679
|
:param start: (optional) Start sensing time in ISO 8601 format (e.g. "1990-11-26",
|
|
1642
1680
|
"1990-11-26T14:30:10.153Z", "1990-11-26T14:30:10+02:00", ...).
|
|
1643
1681
|
If no time offset is given, the time is assumed to be given in UTC.
|
|
1644
|
-
:type start: str
|
|
1645
1682
|
:param end: (optional) End sensing time in ISO 8601 format (e.g. "1990-11-26",
|
|
1646
1683
|
"1990-11-26T14:30:10.153Z", "1990-11-26T14:30:10+02:00", ...).
|
|
1647
1684
|
If no time offset is given, the time is assumed to be given in UTC.
|
|
1648
|
-
:type end: str
|
|
1649
1685
|
:param geom: (optional) Search area that can be defined in different ways (see search)
|
|
1650
|
-
:type geom: Union[str, dict, shapely.geometry.base.BaseGeometry]
|
|
1651
1686
|
:param locations: (optional) Location filtering by name using locations configuration
|
|
1652
|
-
:type locations: dict
|
|
1653
1687
|
:param provider: provider to be used, if no provider is given or the product type
|
|
1654
1688
|
is not available for the provider, the preferred provider is used
|
|
1655
|
-
:type provider: str
|
|
1656
1689
|
:param kwargs: Some other criteria
|
|
1657
1690
|
* id and/or a provider for a search by
|
|
1658
1691
|
* search criteria to guess the product type
|
|
1659
1692
|
* other criteria compatible with the provider
|
|
1660
|
-
:type kwargs: Any
|
|
1661
1693
|
:returns: Search plugins list and the prepared kwargs to make a query.
|
|
1662
|
-
:rtype: tuple(list, dict)
|
|
1663
1694
|
"""
|
|
1664
1695
|
product_type = kwargs.get("productType", None)
|
|
1665
1696
|
if product_type is None:
|
|
@@ -1725,18 +1756,15 @@ class EODataAccessGateway:
|
|
|
1725
1756
|
product_type
|
|
1726
1757
|
not in self._plugins_manager.product_type_to_provider_config_map.keys()
|
|
1727
1758
|
):
|
|
1728
|
-
logger.debug(
|
|
1729
|
-
f"Fetching external product types sources to find {product_type} product type"
|
|
1730
|
-
)
|
|
1731
1759
|
if provider:
|
|
1732
1760
|
# Try to get specific product type from external provider
|
|
1761
|
+
logger.debug(f"Fetching {provider} to find {product_type} product type")
|
|
1733
1762
|
self._fetch_external_product_type(provider, product_type)
|
|
1734
|
-
if
|
|
1735
|
-
not provider
|
|
1736
|
-
or product_type
|
|
1737
|
-
not in self._plugins_manager.product_type_to_provider_config_map.keys()
|
|
1738
|
-
):
|
|
1763
|
+
if not provider:
|
|
1739
1764
|
# no provider or still not found -> fetch all external product types
|
|
1765
|
+
logger.debug(
|
|
1766
|
+
f"Fetching external product types sources to find {product_type} product type"
|
|
1767
|
+
)
|
|
1740
1768
|
self.fetch_product_types_list()
|
|
1741
1769
|
|
|
1742
1770
|
preferred_provider = self.get_preferred_provider()[0]
|
|
@@ -1759,12 +1787,10 @@ class EODataAccessGateway:
|
|
|
1759
1787
|
provider = preferred_provider
|
|
1760
1788
|
providers = [plugin.provider for plugin in search_plugins]
|
|
1761
1789
|
if provider not in providers:
|
|
1762
|
-
logger.
|
|
1763
|
-
"Product type '%s' is not available with provider '%s'.
|
|
1764
|
-
"Searching it on provider '%s' instead.",
|
|
1790
|
+
logger.debug(
|
|
1791
|
+
"Product type '%s' is not available with preferred provider '%s'.",
|
|
1765
1792
|
product_type,
|
|
1766
1793
|
provider,
|
|
1767
|
-
search_plugins[0].provider,
|
|
1768
1794
|
)
|
|
1769
1795
|
else:
|
|
1770
1796
|
provider_plugin = list(
|
|
@@ -1772,11 +1798,6 @@ class EODataAccessGateway:
|
|
|
1772
1798
|
)[0]
|
|
1773
1799
|
search_plugins.remove(provider_plugin)
|
|
1774
1800
|
search_plugins.insert(0, provider_plugin)
|
|
1775
|
-
logger.info(
|
|
1776
|
-
"Searching product type '%s' on provider: %s",
|
|
1777
|
-
product_type,
|
|
1778
|
-
search_plugins[0].provider,
|
|
1779
|
-
)
|
|
1780
1801
|
# Add product_types_config to plugin config. This dict contains product
|
|
1781
1802
|
# type metadata that will also be stored in each product's properties.
|
|
1782
1803
|
for search_plugin in search_plugins:
|
|
@@ -1814,17 +1835,13 @@ class EODataAccessGateway:
|
|
|
1814
1835
|
"""Internal method that performs a search on a given provider.
|
|
1815
1836
|
|
|
1816
1837
|
:param search_plugin: A search plugin
|
|
1817
|
-
:type search_plugin: eodag.plugins.base.Search
|
|
1818
1838
|
:param count: (optional) Whether to run a query with a count request or not
|
|
1819
|
-
:type count: bool
|
|
1820
1839
|
:param raise_errors: (optional) When an error occurs when searching, if this is set to
|
|
1821
1840
|
True, the error is raised
|
|
1822
|
-
:type raise_errors: bool
|
|
1823
1841
|
:param kwargs: Some other criteria that will be used to do the search
|
|
1824
|
-
:type kwargs: Any
|
|
1825
1842
|
:returns: A collection of EO products matching the criteria
|
|
1826
|
-
:rtype: tuple(:class:`~eodag.api.search_result.SearchResult`, int or None)
|
|
1827
1843
|
"""
|
|
1844
|
+
logger.info("Searching on provider %s", search_plugin.provider)
|
|
1828
1845
|
max_items_per_page = getattr(search_plugin.config, "pagination", {}).get(
|
|
1829
1846
|
"max_items_per_page", DEFAULT_MAX_ITEMS_PER_PAGE
|
|
1830
1847
|
)
|
|
@@ -1842,19 +1859,23 @@ class EODataAccessGateway:
|
|
|
1842
1859
|
max_items_per_page,
|
|
1843
1860
|
)
|
|
1844
1861
|
|
|
1845
|
-
need_auth = getattr(search_plugin.config, "need_auth", False)
|
|
1846
|
-
auth_plugin = self._plugins_manager.get_auth_plugin(search_plugin.provider)
|
|
1847
|
-
can_authenticate = callable(getattr(auth_plugin, "authenticate", None))
|
|
1848
|
-
|
|
1849
1862
|
results: List[EOProduct] = []
|
|
1850
1863
|
total_results: Optional[int] = 0 if count else None
|
|
1851
1864
|
|
|
1865
|
+
errors: List[Tuple[str, Exception]] = []
|
|
1866
|
+
|
|
1852
1867
|
try:
|
|
1853
1868
|
prep = PreparedSearch(count=count)
|
|
1854
|
-
if need_auth and auth_plugin and can_authenticate:
|
|
1855
|
-
prep.auth = auth_plugin.authenticate()
|
|
1856
1869
|
|
|
1857
|
-
|
|
1870
|
+
# append auth if needed
|
|
1871
|
+
if getattr(search_plugin.config, "need_auth", False):
|
|
1872
|
+
if auth := self._plugins_manager.get_auth(
|
|
1873
|
+
search_plugin.provider,
|
|
1874
|
+
getattr(search_plugin.config, "api_endpoint", None),
|
|
1875
|
+
search_plugin.config,
|
|
1876
|
+
):
|
|
1877
|
+
prep.auth = auth
|
|
1878
|
+
|
|
1858
1879
|
prep.page = kwargs.pop("page", None)
|
|
1859
1880
|
prep.items_per_page = kwargs.pop("items_per_page", None)
|
|
1860
1881
|
|
|
@@ -1881,6 +1902,7 @@ class EODataAccessGateway:
|
|
|
1881
1902
|
pattern = re.compile(r"[^\w,]+")
|
|
1882
1903
|
try:
|
|
1883
1904
|
guesses = self.guess_product_type(
|
|
1905
|
+
intersect=False,
|
|
1884
1906
|
**{
|
|
1885
1907
|
k: pattern.sub("", str(v).upper())
|
|
1886
1908
|
for k, v in eo_product.properties.items()
|
|
@@ -1894,7 +1916,7 @@ class EODataAccessGateway:
|
|
|
1894
1916
|
"keywords",
|
|
1895
1917
|
]
|
|
1896
1918
|
and v is not None
|
|
1897
|
-
}
|
|
1919
|
+
},
|
|
1898
1920
|
)
|
|
1899
1921
|
except NoMatchingProductType:
|
|
1900
1922
|
pass
|
|
@@ -1907,12 +1929,31 @@ class EODataAccessGateway:
|
|
|
1907
1929
|
eo_product.product_type
|
|
1908
1930
|
)
|
|
1909
1931
|
except NoMatchingProductType:
|
|
1910
|
-
logger.
|
|
1932
|
+
logger.debug("product type %s not found", eo_product.product_type)
|
|
1911
1933
|
|
|
1912
1934
|
if eo_product.search_intersection is not None:
|
|
1913
1935
|
download_plugin = self._plugins_manager.get_download_plugin(
|
|
1914
1936
|
eo_product
|
|
1915
1937
|
)
|
|
1938
|
+
if len(eo_product.assets) > 0:
|
|
1939
|
+
matching_url = next(iter(eo_product.assets.values()))["href"]
|
|
1940
|
+
elif eo_product.properties.get("storageStatus") != ONLINE_STATUS:
|
|
1941
|
+
matching_url = eo_product.properties.get(
|
|
1942
|
+
"orderLink"
|
|
1943
|
+
) or eo_product.properties.get("downloadLink")
|
|
1944
|
+
else:
|
|
1945
|
+
matching_url = eo_product.properties.get("downloadLink")
|
|
1946
|
+
|
|
1947
|
+
try:
|
|
1948
|
+
auth_plugin = next(
|
|
1949
|
+
self._plugins_manager.get_auth_plugins(
|
|
1950
|
+
search_plugin.provider,
|
|
1951
|
+
matching_url=matching_url,
|
|
1952
|
+
matching_conf=download_plugin.config,
|
|
1953
|
+
)
|
|
1954
|
+
)
|
|
1955
|
+
except StopIteration:
|
|
1956
|
+
auth_plugin = None
|
|
1916
1957
|
eo_product.register_downloader(download_plugin, auth_plugin)
|
|
1917
1958
|
|
|
1918
1959
|
results.extend(res)
|
|
@@ -1942,13 +1983,6 @@ class EODataAccessGateway:
|
|
|
1942
1983
|
"the total number of products matching the search criteria"
|
|
1943
1984
|
)
|
|
1944
1985
|
except Exception as e:
|
|
1945
|
-
log_msg = f"No result from provider '{search_plugin.provider}' due to an error during search."
|
|
1946
|
-
if not raise_errors:
|
|
1947
|
-
log_msg += " Raise verbosity of log messages for details"
|
|
1948
|
-
logger.info(log_msg)
|
|
1949
|
-
# keep only the message from exception args
|
|
1950
|
-
if len(e.args) > 1:
|
|
1951
|
-
e.args = (e.args[0],)
|
|
1952
1986
|
if raise_errors:
|
|
1953
1987
|
# Raise the error, letting the application wrapping eodag know that
|
|
1954
1988
|
# something went bad. This way it will be able to decide what to do next
|
|
@@ -1958,16 +1992,14 @@ class EODataAccessGateway:
|
|
|
1958
1992
|
"Error while searching on provider %s (ignored):",
|
|
1959
1993
|
search_plugin.provider,
|
|
1960
1994
|
)
|
|
1961
|
-
|
|
1962
|
-
return SearchResult(results, total_results)
|
|
1995
|
+
errors.append((search_plugin.provider, e))
|
|
1996
|
+
return SearchResult(results, total_results, errors)
|
|
1963
1997
|
|
|
1964
1998
|
def crunch(self, results: SearchResult, **kwargs: Any) -> SearchResult:
|
|
1965
1999
|
"""Apply the filters given through the keyword arguments to the results
|
|
1966
2000
|
|
|
1967
2001
|
:param results: The results of a eodag search request
|
|
1968
|
-
:type results: :class:`~eodag.api.search_result.SearchResult`
|
|
1969
2002
|
:returns: The result of successively applying all the filters to the results
|
|
1970
|
-
:rtype: :class:`~eodag.api.search_result.SearchResult`
|
|
1971
2003
|
"""
|
|
1972
2004
|
search_criteria = kwargs.pop("search_criteria", {})
|
|
1973
2005
|
for cruncher_name, cruncher_args in kwargs.items():
|
|
@@ -1983,7 +2015,6 @@ class EODataAccessGateway:
|
|
|
1983
2015
|
by extent (i.e. bounding box).
|
|
1984
2016
|
|
|
1985
2017
|
:param searches: List of eodag SearchResult
|
|
1986
|
-
:type searches: list
|
|
1987
2018
|
:returns: list of :class:`~eodag.api.search_result.SearchResult`
|
|
1988
2019
|
"""
|
|
1989
2020
|
# Dict with extents as keys, each extent being defined by a str
|
|
@@ -2014,33 +2045,32 @@ class EODataAccessGateway:
|
|
|
2014
2045
|
"""Download all products resulting from a search.
|
|
2015
2046
|
|
|
2016
2047
|
:param search_result: A collection of EO products resulting from a search
|
|
2017
|
-
:type search_result: :class:`~eodag.api.search_result.SearchResult`
|
|
2018
2048
|
:param downloaded_callback: (optional) A method or a callable object which takes
|
|
2019
2049
|
as parameter the ``product``. You can use the base class
|
|
2020
|
-
:class:`~eodag.
|
|
2050
|
+
:class:`~eodag.utils.DownloadedCallback` and override
|
|
2021
2051
|
its ``__call__`` method. Will be called each time a product
|
|
2022
2052
|
finishes downloading
|
|
2023
|
-
:type downloaded_callback: Callable[[:class:`~eodag.api.product._product.EOProduct`], None]
|
|
2024
|
-
or None
|
|
2025
2053
|
:param progress_callback: (optional) A method or a callable object
|
|
2026
2054
|
which takes a current size and a maximum
|
|
2027
2055
|
size as inputs and handle progress bar
|
|
2028
2056
|
creation and update to give the user a
|
|
2029
2057
|
feedback on the download progress
|
|
2030
|
-
:type progress_callback: :class:`~eodag.utils.ProgressCallback` or None
|
|
2031
2058
|
:param wait: (optional) If download fails, wait time in minutes between
|
|
2032
2059
|
two download tries of the same product
|
|
2033
|
-
:type wait: int
|
|
2034
2060
|
:param timeout: (optional) If download fails, maximum time in minutes
|
|
2035
2061
|
before stop retrying to download
|
|
2036
|
-
:
|
|
2037
|
-
|
|
2038
|
-
|
|
2039
|
-
|
|
2040
|
-
|
|
2041
|
-
|
|
2062
|
+
:param kwargs: Additional keyword arguments from the download plugin configuration class that can
|
|
2063
|
+
be provided to override any other values defined in a configuration file
|
|
2064
|
+
or with environment variables:
|
|
2065
|
+
|
|
2066
|
+
* ``output_dir`` - where to store downloaded products, as an absolute file path
|
|
2067
|
+
(Default: local temporary directory)
|
|
2068
|
+
* ``output_extension`` - downloaded file extension
|
|
2069
|
+
* ``extract`` - whether to extract the downloaded products, only applies to archived products
|
|
2070
|
+
* ``dl_url_params`` - additional parameters to pass over to the download url as an url parameter
|
|
2071
|
+
* ``delete_archive`` - whether to delete the downloaded archives
|
|
2072
|
+
* ``asset`` - regex filter to identify assets to download
|
|
2042
2073
|
:returns: A collection of the absolute paths to the downloaded products
|
|
2043
|
-
:rtype: list
|
|
2044
2074
|
"""
|
|
2045
2075
|
paths = []
|
|
2046
2076
|
if search_result:
|
|
@@ -2069,11 +2099,8 @@ class EODataAccessGateway:
|
|
|
2069
2099
|
"""Registers results of a search into a geojson file.
|
|
2070
2100
|
|
|
2071
2101
|
:param search_result: A collection of EO products resulting from a search
|
|
2072
|
-
:type search_result: :class:`~eodag.api.search_result.SearchResult`
|
|
2073
2102
|
:param filename: (optional) The name of the file to generate
|
|
2074
|
-
:type filename: str
|
|
2075
2103
|
:returns: The name of the created file
|
|
2076
|
-
:rtype: str
|
|
2077
2104
|
"""
|
|
2078
2105
|
with open(filename, "w") as fh:
|
|
2079
2106
|
geojson.dump(search_result, fh)
|
|
@@ -2084,9 +2111,7 @@ class EODataAccessGateway:
|
|
|
2084
2111
|
"""Loads results of a search from a geojson file.
|
|
2085
2112
|
|
|
2086
2113
|
:param filename: A filename containing a search result encoded as a geojson
|
|
2087
|
-
:type filename: str
|
|
2088
2114
|
:returns: The search results encoded in `filename`
|
|
2089
|
-
:rtype: :class:`~eodag.api.search_result.SearchResult`
|
|
2090
2115
|
"""
|
|
2091
2116
|
with open(filename, "r") as fh:
|
|
2092
2117
|
return SearchResult.from_geojson(geojson.load(fh))
|
|
@@ -2096,19 +2121,17 @@ class EODataAccessGateway:
|
|
|
2096
2121
|
products with the information needed to download itself
|
|
2097
2122
|
|
|
2098
2123
|
:param filename: A filename containing a search result encoded as a geojson
|
|
2099
|
-
:type filename: str
|
|
2100
2124
|
:returns: The search results encoded in `filename`
|
|
2101
|
-
:rtype: :class:`~eodag.api.search_result.SearchResult`
|
|
2102
2125
|
"""
|
|
2103
2126
|
products = self.deserialize(filename)
|
|
2104
2127
|
for i, product in enumerate(products):
|
|
2105
2128
|
if product.downloader is None:
|
|
2129
|
+
downloader = self._plugins_manager.get_download_plugin(product)
|
|
2106
2130
|
auth = product.downloader_auth
|
|
2107
2131
|
if auth is None:
|
|
2108
|
-
auth = self._plugins_manager.get_auth_plugin(product
|
|
2109
|
-
products[i].register_downloader(
|
|
2110
|
-
|
|
2111
|
-
)
|
|
2132
|
+
auth = self._plugins_manager.get_auth_plugin(downloader, product)
|
|
2133
|
+
products[i].register_downloader(downloader, auth)
|
|
2134
|
+
|
|
2112
2135
|
return products
|
|
2113
2136
|
|
|
2114
2137
|
@_deprecated(
|
|
@@ -2131,22 +2154,14 @@ class EODataAccessGateway:
|
|
|
2131
2154
|
the response content to an API request.
|
|
2132
2155
|
|
|
2133
2156
|
:param filename: A filename containing features encoded as a geojson
|
|
2134
|
-
:type filename: str
|
|
2135
2157
|
:param recursive: (optional) Browse recursively in child nodes if True
|
|
2136
|
-
:
|
|
2137
|
-
:param max_connections: (optional) Maximum number of connections for HTTP requests
|
|
2138
|
-
:type max_connections: int
|
|
2158
|
+
:param max_connections: (optional) Maximum number of connections for concurrent HTTP requests
|
|
2139
2159
|
:param provider: (optional) Data provider
|
|
2140
|
-
:type provider: str
|
|
2141
2160
|
:param productType: (optional) Data product type
|
|
2142
|
-
:type productType: str
|
|
2143
2161
|
:param timeout: (optional) Timeout in seconds for each internal HTTP request
|
|
2144
|
-
:type timeout: float
|
|
2145
2162
|
:param kwargs: Parameters that will be stored in the result as
|
|
2146
2163
|
search criteria
|
|
2147
|
-
:type kwargs: Any
|
|
2148
2164
|
:returns: The search results encoded in `filename`
|
|
2149
|
-
:rtype: :class:`~eodag.api.search_result.SearchResult`
|
|
2150
2165
|
|
|
2151
2166
|
.. deprecated:: 2.2.1
|
|
2152
2167
|
Use the :class:`~eodag.plugins.search.static_stac_search.StaticStacSearch` search plugin instead.
|
|
@@ -2211,26 +2226,27 @@ class EODataAccessGateway:
|
|
|
2211
2226
|
trying to download the product.
|
|
2212
2227
|
|
|
2213
2228
|
:param product: The EO product to download
|
|
2214
|
-
:type product: :class:`~eodag.api.product._product.EOProduct`
|
|
2215
2229
|
:param progress_callback: (optional) A method or a callable object
|
|
2216
2230
|
which takes a current size and a maximum
|
|
2217
2231
|
size as inputs and handle progress bar
|
|
2218
2232
|
creation and update to give the user a
|
|
2219
2233
|
feedback on the download progress
|
|
2220
|
-
:type progress_callback: :class:`~eodag.utils.ProgressCallback` or None
|
|
2221
2234
|
:param wait: (optional) If download fails, wait time in minutes between
|
|
2222
2235
|
two download tries
|
|
2223
|
-
:type wait: int
|
|
2224
2236
|
:param timeout: (optional) If download fails, maximum time in minutes
|
|
2225
2237
|
before stop retrying to download
|
|
2226
|
-
:
|
|
2227
|
-
|
|
2228
|
-
|
|
2229
|
-
|
|
2230
|
-
|
|
2231
|
-
|
|
2238
|
+
:param kwargs: Additional keyword arguments from the download plugin configuration class that can
|
|
2239
|
+
be provided to override any other values defined in a configuration file
|
|
2240
|
+
or with environment variables:
|
|
2241
|
+
|
|
2242
|
+
* ``output_dir`` - where to store downloaded products, as an absolute file path
|
|
2243
|
+
(Default: local temporary directory)
|
|
2244
|
+
* ``output_extension`` - downloaded file extension
|
|
2245
|
+
* ``extract`` - whether to extract the downloaded products, only applies to archived products
|
|
2246
|
+
* ``dl_url_params`` - additional parameters to pass over to the download url as an url parameter
|
|
2247
|
+
* ``delete_archive`` - whether to delete the downloaded archives
|
|
2248
|
+
* ``asset`` - regex filter to identify assets to download
|
|
2232
2249
|
:returns: The absolute path to the downloaded product in the local filesystem
|
|
2233
|
-
:rtype: str
|
|
2234
2250
|
:raises: :class:`~eodag.utils.exceptions.PluginImplementationError`
|
|
2235
2251
|
:raises: :class:`RuntimeError`
|
|
2236
2252
|
"""
|
|
@@ -2246,22 +2262,18 @@ class EODataAccessGateway:
|
|
|
2246
2262
|
|
|
2247
2263
|
def _setup_downloader(self, product: EOProduct) -> None:
|
|
2248
2264
|
if product.downloader is None:
|
|
2265
|
+
downloader = self._plugins_manager.get_download_plugin(product)
|
|
2249
2266
|
auth = product.downloader_auth
|
|
2250
2267
|
if auth is None:
|
|
2251
|
-
auth = self._plugins_manager.get_auth_plugin(product
|
|
2252
|
-
product.register_downloader(
|
|
2253
|
-
self._plugins_manager.get_download_plugin(product), auth
|
|
2254
|
-
)
|
|
2268
|
+
auth = self._plugins_manager.get_auth_plugin(downloader, product)
|
|
2269
|
+
product.register_downloader(downloader, auth)
|
|
2255
2270
|
|
|
2256
2271
|
def get_cruncher(self, name: str, **options: Any) -> Crunch:
|
|
2257
2272
|
"""Build a crunch plugin from a configuration
|
|
2258
2273
|
|
|
2259
2274
|
:param name: The name of the cruncher to build
|
|
2260
|
-
:type name: str
|
|
2261
2275
|
:param options: The configuration options of the cruncher
|
|
2262
|
-
:type options: dict
|
|
2263
2276
|
:returns: The cruncher named ``name``
|
|
2264
|
-
:rtype: :class:`~eodag.plugins.crunch.Crunch`
|
|
2265
2277
|
"""
|
|
2266
2278
|
plugin_conf = {"name": name}
|
|
2267
2279
|
plugin_conf.update({key.replace("-", "_"): val for key, val in options.items()})
|
|
@@ -2273,17 +2285,14 @@ class EODataAccessGateway:
|
|
|
2273
2285
|
"""Fetch the queryable properties for a given product type and/or provider.
|
|
2274
2286
|
|
|
2275
2287
|
:param provider: (optional) The provider.
|
|
2276
|
-
:type provider: str
|
|
2277
2288
|
:param kwargs: additional filters for queryables (`productType` or other search
|
|
2278
2289
|
arguments)
|
|
2279
|
-
:type kwargs: Any
|
|
2280
2290
|
|
|
2281
2291
|
:raises UnsupportedProductType: If the specified product type is not available for the
|
|
2282
2292
|
provider.
|
|
2283
2293
|
|
|
2284
2294
|
:returns: A dict containing the EODAG queryable properties, associating
|
|
2285
2295
|
parameters to their annotated type
|
|
2286
|
-
:rtype: Dict[str, Annotated[Any, FieldInfo]]
|
|
2287
2296
|
"""
|
|
2288
2297
|
available_product_types = [
|
|
2289
2298
|
pt["ID"]
|
|
@@ -2309,7 +2318,7 @@ class EODataAccessGateway:
|
|
|
2309
2318
|
|
|
2310
2319
|
for plugin in self._plugins_manager.get_search_plugins(product_type, provider):
|
|
2311
2320
|
if getattr(plugin.config, "need_auth", False) and (
|
|
2312
|
-
auth := self._plugins_manager.get_auth_plugin(plugin
|
|
2321
|
+
auth := self._plugins_manager.get_auth_plugin(plugin)
|
|
2313
2322
|
):
|
|
2314
2323
|
plugin.auth = auth.authenticate()
|
|
2315
2324
|
providers_queryables[plugin.provider] = plugin.list_queryables(
|
|
@@ -2339,9 +2348,8 @@ class EODataAccessGateway:
|
|
|
2339
2348
|
"""For each provider, gives its available sortable parameter(s) and its maximum
|
|
2340
2349
|
number of them if it supports the sorting feature, otherwise gives None.
|
|
2341
2350
|
|
|
2342
|
-
:returns: A
|
|
2351
|
+
:returns: A dictionary with providers as keys and dictionary of sortable parameter(s) and
|
|
2343
2352
|
its (their) maximum number as value(s).
|
|
2344
|
-
:rtype: dict
|
|
2345
2353
|
:raises: :class:`~eodag.utils.exceptions.UnsupportedProvider`
|
|
2346
2354
|
"""
|
|
2347
2355
|
sortables: Dict[str, Optional[ProviderSortables]] = {}
|