eodag 3.9.1__py3-none-any.whl → 4.0.0a1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- eodag/api/core.py +378 -419
- eodag/api/product/__init__.py +3 -3
- eodag/api/product/_product.py +68 -40
- eodag/api/product/drivers/__init__.py +3 -5
- eodag/api/product/drivers/base.py +1 -18
- eodag/api/product/metadata_mapping.py +151 -215
- eodag/api/search_result.py +13 -7
- eodag/cli.py +72 -395
- eodag/config.py +46 -50
- eodag/plugins/apis/base.py +2 -2
- eodag/plugins/apis/ecmwf.py +20 -21
- eodag/plugins/apis/usgs.py +37 -33
- eodag/plugins/authentication/aws_auth.py +36 -1
- eodag/plugins/authentication/base.py +18 -3
- eodag/plugins/authentication/sas_auth.py +15 -0
- eodag/plugins/crunch/filter_date.py +3 -3
- eodag/plugins/crunch/filter_latest_intersect.py +2 -2
- eodag/plugins/crunch/filter_latest_tpl_name.py +1 -1
- eodag/plugins/download/aws.py +45 -41
- eodag/plugins/download/base.py +13 -14
- eodag/plugins/download/http.py +65 -65
- eodag/plugins/manager.py +28 -29
- eodag/plugins/search/__init__.py +3 -4
- eodag/plugins/search/base.py +128 -77
- eodag/plugins/search/build_search_result.py +105 -107
- eodag/plugins/search/cop_marine.py +44 -47
- eodag/plugins/search/csw.py +33 -33
- eodag/plugins/search/qssearch.py +335 -354
- eodag/plugins/search/stac_list_assets.py +1 -1
- eodag/plugins/search/static_stac_search.py +31 -31
- eodag/resources/{product_types.yml → collections.yml} +2353 -2429
- eodag/resources/ext_collections.json +1 -0
- eodag/resources/ext_product_types.json +1 -1
- eodag/resources/providers.yml +2432 -2714
- eodag/resources/stac_provider.yml +46 -90
- eodag/types/queryables.py +55 -91
- eodag/types/search_args.py +1 -1
- eodag/utils/__init__.py +94 -21
- eodag/utils/exceptions.py +6 -6
- eodag/utils/free_text_search.py +3 -3
- {eodag-3.9.1.dist-info → eodag-4.0.0a1.dist-info}/METADATA +11 -88
- eodag-4.0.0a1.dist-info/RECORD +92 -0
- {eodag-3.9.1.dist-info → eodag-4.0.0a1.dist-info}/entry_points.txt +0 -4
- eodag/plugins/authentication/oauth.py +0 -60
- eodag/plugins/download/creodias_s3.py +0 -64
- eodag/plugins/download/s3rest.py +0 -351
- eodag/plugins/search/data_request_search.py +0 -565
- eodag/resources/stac.yml +0 -294
- eodag/resources/stac_api.yml +0 -2105
- eodag/rest/__init__.py +0 -24
- eodag/rest/cache.py +0 -70
- eodag/rest/config.py +0 -67
- eodag/rest/constants.py +0 -26
- eodag/rest/core.py +0 -764
- eodag/rest/errors.py +0 -210
- eodag/rest/server.py +0 -604
- eodag/rest/server.wsgi +0 -6
- eodag/rest/stac.py +0 -1032
- eodag/rest/templates/README +0 -1
- eodag/rest/types/__init__.py +0 -18
- eodag/rest/types/collections_search.py +0 -44
- eodag/rest/types/eodag_search.py +0 -386
- eodag/rest/types/queryables.py +0 -174
- eodag/rest/types/stac_search.py +0 -272
- eodag/rest/utils/__init__.py +0 -207
- eodag/rest/utils/cql_evaluate.py +0 -119
- eodag/rest/utils/rfc3339.py +0 -64
- eodag-3.9.1.dist-info/RECORD +0 -115
- {eodag-3.9.1.dist-info → eodag-4.0.0a1.dist-info}/WHEEL +0 -0
- {eodag-3.9.1.dist-info → eodag-4.0.0a1.dist-info}/licenses/LICENSE +0 -0
- {eodag-3.9.1.dist-info → eodag-4.0.0a1.dist-info}/top_level.txt +0 -0
eodag/api/core.py
CHANGED
|
@@ -41,7 +41,7 @@ from eodag.config import (
|
|
|
41
41
|
PluginConfig,
|
|
42
42
|
SimpleYamlProxyConfig,
|
|
43
43
|
credentials_in_auth,
|
|
44
|
-
|
|
44
|
+
get_ext_collections_conf,
|
|
45
45
|
load_default_config,
|
|
46
46
|
load_stac_provider_config,
|
|
47
47
|
load_yml_config,
|
|
@@ -56,18 +56,16 @@ from eodag.plugins.search import PreparedSearch
|
|
|
56
56
|
from eodag.plugins.search.build_search_result import MeteoblueSearch
|
|
57
57
|
from eodag.plugins.search.qssearch import PostJsonSearch
|
|
58
58
|
from eodag.types import model_fields_to_annotated
|
|
59
|
-
from eodag.types.queryables import CommonQueryables, QueryablesDict
|
|
59
|
+
from eodag.types.queryables import CommonQueryables, Queryables, QueryablesDict
|
|
60
60
|
from eodag.utils import (
|
|
61
61
|
DEFAULT_DOWNLOAD_TIMEOUT,
|
|
62
62
|
DEFAULT_DOWNLOAD_WAIT,
|
|
63
63
|
DEFAULT_ITEMS_PER_PAGE,
|
|
64
64
|
DEFAULT_MAX_ITEMS_PER_PAGE,
|
|
65
65
|
DEFAULT_PAGE,
|
|
66
|
-
|
|
66
|
+
GENERIC_COLLECTION,
|
|
67
67
|
GENERIC_STAC_PROVIDER,
|
|
68
|
-
|
|
69
|
-
MockResponse,
|
|
70
|
-
_deprecated,
|
|
68
|
+
get_collection_dates,
|
|
71
69
|
get_geometry_from_various,
|
|
72
70
|
makedirs,
|
|
73
71
|
sort_dict,
|
|
@@ -78,10 +76,10 @@ from eodag.utils.dates import rfc3339_str_to_datetime
|
|
|
78
76
|
from eodag.utils.env import is_env_var_true
|
|
79
77
|
from eodag.utils.exceptions import (
|
|
80
78
|
AuthenticationError,
|
|
81
|
-
|
|
79
|
+
NoMatchingCollection,
|
|
82
80
|
PluginImplementationError,
|
|
83
81
|
RequestError,
|
|
84
|
-
|
|
82
|
+
UnsupportedCollection,
|
|
85
83
|
UnsupportedProvider,
|
|
86
84
|
)
|
|
87
85
|
from eodag.utils.free_text_search import compile_free_text_query
|
|
@@ -114,10 +112,10 @@ class EODataAccessGateway:
|
|
|
114
112
|
user_conf_file_path: Optional[str] = None,
|
|
115
113
|
locations_conf_path: Optional[str] = None,
|
|
116
114
|
) -> None:
|
|
117
|
-
|
|
118
|
-
res_files("eodag") / "resources" / "
|
|
115
|
+
collections_config_path = os.getenv("EODAG_COLLECTIONS_CFG_FILE") or str(
|
|
116
|
+
res_files("eodag") / "resources" / "collections.yml"
|
|
119
117
|
)
|
|
120
|
-
self.
|
|
118
|
+
self.collections_config = SimpleYamlProxyConfig(collections_config_path)
|
|
121
119
|
self.providers_config = load_default_config()
|
|
122
120
|
|
|
123
121
|
env_var_cfg_dir = "EODAG_CFG_DIR"
|
|
@@ -169,8 +167,8 @@ class EODataAccessGateway:
|
|
|
169
167
|
share_credentials(self.providers_config)
|
|
170
168
|
|
|
171
169
|
# init updated providers conf
|
|
172
|
-
strict_mode = is_env_var_true("
|
|
173
|
-
|
|
170
|
+
strict_mode = is_env_var_true("EODAG_STRICT_COLLECTIONS")
|
|
171
|
+
available_collections = set(self.collections_config.source.keys())
|
|
174
172
|
|
|
175
173
|
for provider in self.providers_config.keys():
|
|
176
174
|
provider_config_init(
|
|
@@ -178,11 +176,11 @@ class EODataAccessGateway:
|
|
|
178
176
|
load_stac_provider_config(),
|
|
179
177
|
)
|
|
180
178
|
|
|
181
|
-
self.
|
|
182
|
-
provider,
|
|
179
|
+
self._sync_provider_collections(
|
|
180
|
+
provider, available_collections, strict_mode
|
|
183
181
|
)
|
|
184
|
-
# init
|
|
185
|
-
self.
|
|
182
|
+
# init collections configuration
|
|
183
|
+
self._collections_config_init()
|
|
186
184
|
|
|
187
185
|
# re-build _plugins_manager using up-to-date providers_config
|
|
188
186
|
self._plugins_manager.rebuild(self.providers_config)
|
|
@@ -225,26 +223,26 @@ class EODataAccessGateway:
|
|
|
225
223
|
)
|
|
226
224
|
self.set_locations_conf(locations_conf_path)
|
|
227
225
|
|
|
228
|
-
def
|
|
229
|
-
"""Initialize
|
|
230
|
-
for pt_id, pd_dict in self.
|
|
231
|
-
self.
|
|
226
|
+
def _collections_config_init(self) -> None:
|
|
227
|
+
"""Initialize collections configuration."""
|
|
228
|
+
for pt_id, pd_dict in self.collections_config.source.items():
|
|
229
|
+
self.collections_config.source[pt_id].setdefault("_id", pt_id)
|
|
232
230
|
|
|
233
|
-
def
|
|
231
|
+
def _sync_provider_collections(
|
|
234
232
|
self,
|
|
235
233
|
provider: str,
|
|
236
|
-
|
|
234
|
+
available_collections: set[str],
|
|
237
235
|
strict_mode: bool,
|
|
238
236
|
) -> None:
|
|
239
237
|
"""
|
|
240
|
-
Synchronize
|
|
238
|
+
Synchronize collections for a provider based on strict or permissive mode.
|
|
241
239
|
|
|
242
|
-
In strict mode, removes
|
|
243
|
-
In permissive mode, adds empty
|
|
240
|
+
In strict mode, removes collections not in available_collections.
|
|
241
|
+
In permissive mode, adds empty collection configs for missing types.
|
|
244
242
|
|
|
245
|
-
:param provider: The provider name whose
|
|
246
|
-
:param
|
|
247
|
-
:param strict_mode: If True, remove unknown
|
|
243
|
+
:param provider: The provider name whose collections should be synchronized.
|
|
244
|
+
:param available_collections: The set of available collection IDs.
|
|
245
|
+
:param strict_mode: If True, remove unknown collections; if False, add empty configs for them.
|
|
248
246
|
:returns: None
|
|
249
247
|
"""
|
|
250
248
|
provider_products = self.providers_config[provider].products
|
|
@@ -252,33 +250,33 @@ class EODataAccessGateway:
|
|
|
252
250
|
products_to_add: list[str] = []
|
|
253
251
|
|
|
254
252
|
for product_id in provider_products:
|
|
255
|
-
if product_id ==
|
|
253
|
+
if product_id == GENERIC_COLLECTION:
|
|
256
254
|
continue
|
|
257
255
|
|
|
258
|
-
if product_id not in
|
|
256
|
+
if product_id not in available_collections:
|
|
259
257
|
if strict_mode:
|
|
260
258
|
products_to_remove.append(product_id)
|
|
261
259
|
continue
|
|
262
260
|
|
|
263
261
|
empty_product = {
|
|
264
262
|
"title": product_id,
|
|
265
|
-
"
|
|
263
|
+
"description": NOT_AVAILABLE,
|
|
266
264
|
}
|
|
267
|
-
self.
|
|
265
|
+
self.collections_config.source[
|
|
268
266
|
product_id
|
|
269
|
-
] = empty_product # will update
|
|
267
|
+
] = empty_product # will update available_collections
|
|
270
268
|
products_to_add.append(product_id)
|
|
271
269
|
|
|
272
270
|
if products_to_add:
|
|
273
271
|
logger.debug(
|
|
274
|
-
"
|
|
272
|
+
"Collections permissive mode, %s added (provider %s)",
|
|
275
273
|
", ".join(products_to_add),
|
|
276
274
|
provider,
|
|
277
275
|
)
|
|
278
276
|
|
|
279
277
|
if products_to_remove:
|
|
280
278
|
logger.debug(
|
|
281
|
-
"
|
|
279
|
+
"Collections strict mode, ignoring %s (provider %s)",
|
|
282
280
|
", ".join(products_to_remove),
|
|
283
281
|
provider,
|
|
284
282
|
)
|
|
@@ -357,9 +355,9 @@ class EODataAccessGateway:
|
|
|
357
355
|
self.providers_config[provider],
|
|
358
356
|
load_stac_provider_config(),
|
|
359
357
|
)
|
|
360
|
-
setattr(self.providers_config[provider], "
|
|
358
|
+
setattr(self.providers_config[provider], "collections_fetched", False)
|
|
361
359
|
# re-create _plugins_manager using up-to-date providers_config
|
|
362
|
-
self._plugins_manager.
|
|
360
|
+
self._plugins_manager.build_collection_to_provider_config_map()
|
|
363
361
|
|
|
364
362
|
def add_provider(
|
|
365
363
|
self,
|
|
@@ -368,7 +366,7 @@ class EODataAccessGateway:
|
|
|
368
366
|
priority: Optional[int] = None,
|
|
369
367
|
search: dict[str, Any] = {"type": "StacSearch"},
|
|
370
368
|
products: dict[str, Any] = {
|
|
371
|
-
|
|
369
|
+
GENERIC_COLLECTION: {"_collection": "{collection}"}
|
|
372
370
|
},
|
|
373
371
|
download: dict[str, Any] = {"type": "HTTPDownload", "auth_error_code": 401},
|
|
374
372
|
**kwargs: dict[str, Any],
|
|
@@ -379,14 +377,14 @@ class EODataAccessGateway:
|
|
|
379
377
|
updated (not replaced), with user provided ones:
|
|
380
378
|
|
|
381
379
|
* ``search`` : ``{"type": "StacSearch"}``
|
|
382
|
-
* ``products`` : ``{"
|
|
380
|
+
* ``products`` : ``{"GENERIC_COLLECTION": {"_collection": "{collection}"}}``
|
|
383
381
|
* ``download`` : ``{"type": "HTTPDownload", "auth_error_code": 401}``
|
|
384
382
|
|
|
385
383
|
:param name: Name of provider
|
|
386
384
|
:param url: Provider url, also used as ``search["api_endpoint"]`` if not defined
|
|
387
385
|
:param priority: Provider priority. If None, provider will be set as preferred (highest priority)
|
|
388
386
|
:param search: Search :class:`~eodag.config.PluginConfig` mapping
|
|
389
|
-
:param products: Provider
|
|
387
|
+
:param products: Provider collections mapping
|
|
390
388
|
:param download: Download :class:`~eodag.config.PluginConfig` mapping
|
|
391
389
|
:param kwargs: Additional :class:`~eodag.config.ProviderConfig` mapping
|
|
392
390
|
"""
|
|
@@ -395,7 +393,7 @@ class EODataAccessGateway:
|
|
|
395
393
|
"url": url,
|
|
396
394
|
"search": {"type": "StacSearch", **search},
|
|
397
395
|
"products": {
|
|
398
|
-
|
|
396
|
+
GENERIC_COLLECTION: {"_collection": "{collection}"},
|
|
399
397
|
**products,
|
|
400
398
|
},
|
|
401
399
|
"download": {
|
|
@@ -541,23 +539,23 @@ class EODataAccessGateway:
|
|
|
541
539
|
)
|
|
542
540
|
self.locations_config = []
|
|
543
541
|
|
|
544
|
-
def
|
|
542
|
+
def list_collections(
|
|
545
543
|
self, provider: Optional[str] = None, fetch_providers: bool = True
|
|
546
544
|
) -> list[dict[str, Any]]:
|
|
547
|
-
"""Lists supported
|
|
545
|
+
"""Lists supported collections.
|
|
548
546
|
|
|
549
547
|
:param provider: (optional) The name of a provider that must support the product
|
|
550
548
|
types we are about to list
|
|
551
549
|
:param fetch_providers: (optional) Whether to fetch providers for new product
|
|
552
550
|
types or not
|
|
553
|
-
:returns: The list of the
|
|
551
|
+
:returns: The list of the collections that can be accessed using eodag.
|
|
554
552
|
:raises: :class:`~eodag.utils.exceptions.UnsupportedProvider`
|
|
555
553
|
"""
|
|
556
554
|
if fetch_providers:
|
|
557
|
-
# First, update
|
|
558
|
-
self.
|
|
555
|
+
# First, update collections list if possible
|
|
556
|
+
self.fetch_collections_list(provider=provider)
|
|
559
557
|
|
|
560
|
-
|
|
558
|
+
collections: list[dict[str, Any]] = []
|
|
561
559
|
|
|
562
560
|
providers_configs = (
|
|
563
561
|
list(self.providers_config.values())
|
|
@@ -575,31 +573,31 @@ class EODataAccessGateway:
|
|
|
575
573
|
)
|
|
576
574
|
|
|
577
575
|
for p in providers_configs:
|
|
578
|
-
for
|
|
579
|
-
if
|
|
576
|
+
for collection_id in p.products: # type: ignore
|
|
577
|
+
if collection_id == GENERIC_COLLECTION:
|
|
580
578
|
continue
|
|
581
579
|
|
|
582
|
-
config = self.
|
|
580
|
+
config = self.collections_config[collection_id]
|
|
583
581
|
if "alias" in config:
|
|
584
|
-
|
|
585
|
-
|
|
582
|
+
collection_id = config["alias"]
|
|
583
|
+
collection = {"ID": collection_id, **config}
|
|
586
584
|
|
|
587
|
-
if
|
|
588
|
-
|
|
585
|
+
if collection not in collections:
|
|
586
|
+
collections.append(collection)
|
|
589
587
|
|
|
590
|
-
# Return the
|
|
591
|
-
return sorted(
|
|
588
|
+
# Return the collections sorted in lexicographic order of their ID
|
|
589
|
+
return sorted(collections, key=itemgetter("ID"))
|
|
592
590
|
|
|
593
|
-
def
|
|
594
|
-
"""Fetch
|
|
591
|
+
def fetch_collections_list(self, provider: Optional[str] = None) -> None:
|
|
592
|
+
"""Fetch collections list and update if needed.
|
|
595
593
|
|
|
596
|
-
If strict mode is enabled (by setting the ``
|
|
597
|
-
to a truthy value), this method will not fetch or update
|
|
594
|
+
If strict mode is enabled (by setting the ``EODAG_STRICT_COLLECTIONS`` environment variable
|
|
595
|
+
to a truthy value), this method will not fetch or update collections and will return immediately.
|
|
598
596
|
|
|
599
|
-
:param provider: The name of a provider or provider-group for which
|
|
597
|
+
:param provider: The name of a provider or provider-group for which collections
|
|
600
598
|
list should be updated. Defaults to all providers (None value).
|
|
601
599
|
"""
|
|
602
|
-
strict_mode = is_env_var_true("
|
|
600
|
+
strict_mode = is_env_var_true("EODAG_STRICT_COLLECTIONS")
|
|
603
601
|
if strict_mode:
|
|
604
602
|
return
|
|
605
603
|
|
|
@@ -613,7 +611,7 @@ class EODataAccessGateway:
|
|
|
613
611
|
]
|
|
614
612
|
if providers_to_fetch:
|
|
615
613
|
logger.info(
|
|
616
|
-
f"Fetch
|
|
614
|
+
f"Fetch collections for {provider} group: {', '.join(providers_to_fetch)}"
|
|
617
615
|
)
|
|
618
616
|
else:
|
|
619
617
|
return None
|
|
@@ -622,7 +620,7 @@ class EODataAccessGateway:
|
|
|
622
620
|
|
|
623
621
|
# providers discovery confs that are fetchable
|
|
624
622
|
providers_discovery_configs_fetchable: dict[str, Any] = {}
|
|
625
|
-
# check if any provider has not already been fetched for
|
|
623
|
+
# check if any provider has not already been fetched for collections
|
|
626
624
|
already_fetched = True
|
|
627
625
|
for provider_to_fetch in providers_to_fetch:
|
|
628
626
|
provider_config = self.providers_config[provider_to_fetch]
|
|
@@ -633,45 +631,43 @@ class EODataAccessGateway:
|
|
|
633
631
|
provider_search_config = provider_config.api
|
|
634
632
|
else:
|
|
635
633
|
continue
|
|
636
|
-
discovery_conf = getattr(
|
|
637
|
-
provider_search_config, "discover_product_types", {}
|
|
638
|
-
)
|
|
634
|
+
discovery_conf = getattr(provider_search_config, "discover_collections", {})
|
|
639
635
|
if discovery_conf.get("fetch_url"):
|
|
640
636
|
providers_discovery_configs_fetchable[
|
|
641
637
|
provider_to_fetch
|
|
642
638
|
] = discovery_conf
|
|
643
|
-
if not getattr(provider_config, "
|
|
639
|
+
if not getattr(provider_config, "collections_fetched", False):
|
|
644
640
|
already_fetched = False
|
|
645
641
|
|
|
646
642
|
if not already_fetched:
|
|
647
|
-
# get
|
|
648
|
-
|
|
649
|
-
if
|
|
650
|
-
|
|
651
|
-
|
|
643
|
+
# get ext_collections conf
|
|
644
|
+
ext_collections_cfg_file = os.getenv("EODAG_EXT_COLLECTIONS_CFG_FILE")
|
|
645
|
+
if ext_collections_cfg_file is not None:
|
|
646
|
+
ext_collections_conf = get_ext_collections_conf(
|
|
647
|
+
ext_collections_cfg_file
|
|
652
648
|
)
|
|
653
649
|
else:
|
|
654
|
-
|
|
650
|
+
ext_collections_conf = get_ext_collections_conf()
|
|
655
651
|
|
|
656
|
-
if not
|
|
657
|
-
# empty
|
|
658
|
-
|
|
659
|
-
self.
|
|
652
|
+
if not ext_collections_conf:
|
|
653
|
+
# empty ext_collections conf
|
|
654
|
+
ext_collections_conf = (
|
|
655
|
+
self.discover_collections(provider=provider) or {}
|
|
660
656
|
)
|
|
661
657
|
|
|
662
|
-
# update eodag
|
|
663
|
-
self.
|
|
658
|
+
# update eodag collections list with new conf
|
|
659
|
+
self.update_collections_list(ext_collections_conf)
|
|
664
660
|
|
|
665
661
|
# Compare current provider with default one to see if it has been modified
|
|
666
|
-
# and
|
|
662
|
+
# and collections list would need to be fetched
|
|
667
663
|
|
|
668
|
-
# get
|
|
664
|
+
# get ext_collections conf for user modified providers
|
|
669
665
|
default_providers_config = load_default_config()
|
|
670
666
|
for (
|
|
671
667
|
provider,
|
|
672
668
|
user_discovery_conf,
|
|
673
669
|
) in providers_discovery_configs_fetchable.items():
|
|
674
|
-
# default
|
|
670
|
+
# default discover_collections conf
|
|
675
671
|
if provider in default_providers_config:
|
|
676
672
|
default_provider_config = default_providers_config[provider]
|
|
677
673
|
if hasattr(default_provider_config, "search"):
|
|
@@ -681,7 +677,7 @@ class EODataAccessGateway:
|
|
|
681
677
|
else:
|
|
682
678
|
continue
|
|
683
679
|
default_discovery_conf = getattr(
|
|
684
|
-
default_provider_search_config, "
|
|
680
|
+
default_provider_search_config, "discover_collections", {}
|
|
685
681
|
)
|
|
686
682
|
# compare confs
|
|
687
683
|
if default_discovery_conf["result_type"] == "json" and isinstance(
|
|
@@ -696,22 +692,22 @@ class EODataAccessGateway:
|
|
|
696
692
|
},
|
|
697
693
|
**mtd_cfg_as_conversion_and_querypath(
|
|
698
694
|
dict(
|
|
699
|
-
|
|
700
|
-
"
|
|
695
|
+
generic_collection_id=default_discovery_conf[
|
|
696
|
+
"generic_collection_id"
|
|
701
697
|
]
|
|
702
698
|
)
|
|
703
699
|
),
|
|
704
700
|
**dict(
|
|
705
|
-
|
|
701
|
+
generic_collection_parsable_properties=mtd_cfg_as_conversion_and_querypath(
|
|
706
702
|
default_discovery_conf[
|
|
707
|
-
"
|
|
703
|
+
"generic_collection_parsable_properties"
|
|
708
704
|
]
|
|
709
705
|
)
|
|
710
706
|
),
|
|
711
707
|
**dict(
|
|
712
|
-
|
|
708
|
+
generic_collection_parsable_metadata=mtd_cfg_as_conversion_and_querypath(
|
|
713
709
|
default_discovery_conf[
|
|
714
|
-
"
|
|
710
|
+
"generic_collection_parsable_metadata"
|
|
715
711
|
]
|
|
716
712
|
)
|
|
717
713
|
),
|
|
@@ -723,33 +719,33 @@ class EODataAccessGateway:
|
|
|
723
719
|
or user_discovery_conf == default_discovery_conf_parsed
|
|
724
720
|
) and (
|
|
725
721
|
not default_discovery_conf.get("fetch_url")
|
|
726
|
-
or "
|
|
727
|
-
or "
|
|
722
|
+
or "ext_collections_conf" not in locals()
|
|
723
|
+
or "ext_collections_conf" in locals()
|
|
728
724
|
and (
|
|
729
|
-
provider in
|
|
730
|
-
or len(
|
|
725
|
+
provider in ext_collections_conf
|
|
726
|
+
or len(ext_collections_conf.keys()) == 0
|
|
731
727
|
)
|
|
732
728
|
):
|
|
733
729
|
continue
|
|
734
730
|
# providers not skipped here should be user-modified
|
|
735
|
-
# or not in
|
|
731
|
+
# or not in ext_collections_conf (if eodag system conf != eodag conf used for ext_collections_conf)
|
|
736
732
|
|
|
737
733
|
if not already_fetched:
|
|
738
|
-
# discover
|
|
739
|
-
|
|
740
|
-
self.
|
|
734
|
+
# discover collections for user configured provider
|
|
735
|
+
provider_ext_collections_conf = (
|
|
736
|
+
self.discover_collections(provider=provider) or {}
|
|
741
737
|
)
|
|
742
|
-
# update eodag
|
|
743
|
-
self.
|
|
738
|
+
# update eodag collections list with new conf
|
|
739
|
+
self.update_collections_list(provider_ext_collections_conf)
|
|
744
740
|
|
|
745
|
-
def
|
|
741
|
+
def discover_collections(
|
|
746
742
|
self, provider: Optional[str] = None
|
|
747
743
|
) -> Optional[dict[str, Any]]:
|
|
748
|
-
"""Fetch providers for
|
|
744
|
+
"""Fetch providers for collections
|
|
749
745
|
|
|
750
746
|
:param provider: The name of a provider or provider-group to fetch. Defaults to
|
|
751
747
|
all providers (None value).
|
|
752
|
-
:returns: external
|
|
748
|
+
:returns: external collections configuration
|
|
753
749
|
"""
|
|
754
750
|
grouped_providers = [
|
|
755
751
|
p
|
|
@@ -758,13 +754,13 @@ class EODataAccessGateway:
|
|
|
758
754
|
]
|
|
759
755
|
if provider and provider not in self.providers_config and grouped_providers:
|
|
760
756
|
logger.info(
|
|
761
|
-
f"Discover
|
|
757
|
+
f"Discover collections for {provider} group: {', '.join(grouped_providers)}"
|
|
762
758
|
)
|
|
763
759
|
elif provider and provider not in self.providers_config:
|
|
764
760
|
raise UnsupportedProvider(
|
|
765
761
|
f"The requested provider is not (yet) supported: {provider}"
|
|
766
762
|
)
|
|
767
|
-
|
|
763
|
+
ext_collections_conf: dict[str, Any] = {}
|
|
768
764
|
providers_to_fetch = [
|
|
769
765
|
p
|
|
770
766
|
for p in (
|
|
@@ -785,14 +781,14 @@ class EODataAccessGateway:
|
|
|
785
781
|
search_plugin_config = self.providers_config[provider].api
|
|
786
782
|
else:
|
|
787
783
|
return None
|
|
788
|
-
if getattr(search_plugin_config, "
|
|
784
|
+
if getattr(search_plugin_config, "discover_collections", {}).get(
|
|
789
785
|
"fetch_url", None
|
|
790
786
|
):
|
|
791
787
|
search_plugin: Union[Search, Api] = next(
|
|
792
788
|
self._plugins_manager.get_search_plugins(provider=provider)
|
|
793
789
|
)
|
|
794
790
|
# check after plugin init if still fetchable
|
|
795
|
-
if not getattr(search_plugin.config, "
|
|
791
|
+
if not getattr(search_plugin.config, "discover_collections", {}).get(
|
|
796
792
|
"fetch_url"
|
|
797
793
|
):
|
|
798
794
|
continue
|
|
@@ -806,26 +802,26 @@ class EODataAccessGateway:
|
|
|
806
802
|
kwargs["auth"] = auth
|
|
807
803
|
else:
|
|
808
804
|
logger.debug(
|
|
809
|
-
f"Could not authenticate on {provider} for
|
|
805
|
+
f"Could not authenticate on {provider} for collections discovery"
|
|
810
806
|
)
|
|
811
|
-
|
|
807
|
+
ext_collections_conf[provider] = None
|
|
812
808
|
continue
|
|
813
809
|
|
|
814
|
-
|
|
810
|
+
ext_collections_conf[provider] = search_plugin.discover_collections(
|
|
815
811
|
**kwargs
|
|
816
812
|
)
|
|
817
813
|
|
|
818
|
-
return sort_dict(
|
|
814
|
+
return sort_dict(ext_collections_conf)
|
|
819
815
|
|
|
820
|
-
def
|
|
821
|
-
self,
|
|
816
|
+
def update_collections_list(
|
|
817
|
+
self, ext_collections_conf: dict[str, Optional[dict[str, dict[str, Any]]]]
|
|
822
818
|
) -> None:
|
|
823
|
-
"""Update eodag
|
|
819
|
+
"""Update eodag collections list
|
|
824
820
|
|
|
825
|
-
:param
|
|
821
|
+
:param ext_collections_conf: external collections configuration
|
|
826
822
|
"""
|
|
827
|
-
for provider,
|
|
828
|
-
if
|
|
823
|
+
for provider, new_collections_conf in ext_collections_conf.items():
|
|
824
|
+
if new_collections_conf and provider in self.providers_config:
|
|
829
825
|
try:
|
|
830
826
|
search_plugin_config = getattr(
|
|
831
827
|
self.providers_config[provider], "search", None
|
|
@@ -833,77 +829,75 @@ class EODataAccessGateway:
|
|
|
833
829
|
if search_plugin_config is None:
|
|
834
830
|
continue
|
|
835
831
|
if not getattr(
|
|
836
|
-
search_plugin_config, "
|
|
832
|
+
search_plugin_config, "discover_collections", {}
|
|
837
833
|
).get("fetch_url"):
|
|
838
|
-
# conf has been updated and provider
|
|
834
|
+
# conf has been updated and provider collections are no more discoverable
|
|
839
835
|
continue
|
|
840
836
|
provider_products_config = (
|
|
841
837
|
self.providers_config[provider].products or {}
|
|
842
838
|
)
|
|
843
839
|
except UnsupportedProvider:
|
|
844
840
|
logger.debug(
|
|
845
|
-
"Ignoring external
|
|
841
|
+
"Ignoring external collections for unknown provider %s",
|
|
846
842
|
provider,
|
|
847
843
|
)
|
|
848
844
|
continue
|
|
849
|
-
|
|
845
|
+
new_collections: list[str] = []
|
|
850
846
|
for (
|
|
851
|
-
|
|
852
|
-
|
|
853
|
-
) in
|
|
854
|
-
if
|
|
855
|
-
for
|
|
847
|
+
new_collection,
|
|
848
|
+
new_collection_conf,
|
|
849
|
+
) in new_collections_conf["providers_config"].items():
|
|
850
|
+
if new_collection not in provider_products_config:
|
|
851
|
+
for existing_collection in provider_products_config.copy():
|
|
856
852
|
# compare parsed extracted conf (without metadata_mapping entry)
|
|
857
853
|
unparsable_keys = (
|
|
858
|
-
search_plugin_config.
|
|
859
|
-
"
|
|
854
|
+
search_plugin_config.discover_collections.get(
|
|
855
|
+
"generic_collection_unparsable_properties", {}
|
|
860
856
|
).keys()
|
|
861
857
|
)
|
|
862
|
-
|
|
858
|
+
new_parsed_collections_conf = {
|
|
863
859
|
k: v
|
|
864
|
-
for k, v in
|
|
860
|
+
for k, v in new_collection_conf.items()
|
|
865
861
|
if k not in unparsable_keys
|
|
866
862
|
}
|
|
867
863
|
if (
|
|
868
|
-
|
|
869
|
-
<= provider_products_config[
|
|
870
|
-
existing_product_type
|
|
871
|
-
].items()
|
|
864
|
+
new_parsed_collections_conf.items()
|
|
865
|
+
<= provider_products_config[existing_collection].items()
|
|
872
866
|
):
|
|
873
|
-
#
|
|
867
|
+
# new_collections_conf is a subset on an existing conf
|
|
874
868
|
break
|
|
875
869
|
else:
|
|
876
|
-
#
|
|
870
|
+
# new_collection_conf does not already exist, append it
|
|
877
871
|
# to provider_products_config
|
|
878
872
|
provider_products_config[
|
|
879
|
-
|
|
880
|
-
] =
|
|
881
|
-
# to self.
|
|
882
|
-
self.
|
|
873
|
+
new_collection
|
|
874
|
+
] = new_collection_conf
|
|
875
|
+
# to self.collections_config
|
|
876
|
+
self.collections_config.source.update(
|
|
883
877
|
{
|
|
884
|
-
|
|
885
|
-
|
|
|
886
|
-
|
|
878
|
+
new_collection: {"_id": new_collection}
|
|
879
|
+
| new_collections_conf["collections_config"][
|
|
880
|
+
new_collection
|
|
887
881
|
]
|
|
888
882
|
}
|
|
889
883
|
)
|
|
890
|
-
|
|
891
|
-
|
|
892
|
-
if
|
|
884
|
+
ext_collections_conf[provider] = new_collections_conf
|
|
885
|
+
new_collections.append(new_collection)
|
|
886
|
+
if new_collections:
|
|
893
887
|
logger.debug(
|
|
894
|
-
f"Added {len(
|
|
888
|
+
f"Added {len(new_collections)} collections for {provider}"
|
|
895
889
|
)
|
|
896
890
|
|
|
897
891
|
elif provider not in self.providers_config:
|
|
898
892
|
# unknown provider
|
|
899
893
|
continue
|
|
900
|
-
self.providers_config[provider].
|
|
894
|
+
self.providers_config[provider].collections_fetched = True
|
|
901
895
|
|
|
902
896
|
# re-create _plugins_manager using up-to-date providers_config
|
|
903
|
-
self._plugins_manager.
|
|
897
|
+
self._plugins_manager.build_collection_to_provider_config_map()
|
|
904
898
|
|
|
905
899
|
def available_providers(
|
|
906
|
-
self,
|
|
900
|
+
self, collection: Optional[str] = None, by_group: bool = False
|
|
907
901
|
) -> list[str]:
|
|
908
902
|
"""Gives the sorted list of the available providers or groups
|
|
909
903
|
|
|
@@ -911,17 +905,17 @@ class EODataAccessGateway:
|
|
|
911
905
|
and then alphabetically in ascending order for providers or groups with the same
|
|
912
906
|
priority level.
|
|
913
907
|
|
|
914
|
-
:param
|
|
908
|
+
:param collection: (optional) Only list providers configured for this collection
|
|
915
909
|
:param by_group: (optional) If set to True, list groups when available instead
|
|
916
910
|
of providers, mixed with other providers
|
|
917
911
|
:returns: the sorted list of the available providers or groups
|
|
918
912
|
"""
|
|
919
913
|
|
|
920
|
-
if
|
|
914
|
+
if collection:
|
|
921
915
|
providers = [
|
|
922
916
|
(v.group if by_group and hasattr(v, "group") else k, v.priority)
|
|
923
917
|
for k, v in self.providers_config.items()
|
|
924
|
-
if
|
|
918
|
+
if collection in getattr(v, "products", {}).keys()
|
|
925
919
|
]
|
|
926
920
|
else:
|
|
927
921
|
providers = [
|
|
@@ -943,66 +937,66 @@ class EODataAccessGateway:
|
|
|
943
937
|
# Return only the names of the providers or groups
|
|
944
938
|
return [name for name, _ in providers]
|
|
945
939
|
|
|
946
|
-
def
|
|
947
|
-
"""Return the ID of a
|
|
940
|
+
def get_collection_from_alias(self, alias_or_id: str) -> str:
|
|
941
|
+
"""Return the ID of a collection by either its ID or alias
|
|
948
942
|
|
|
949
|
-
:param alias_or_id: Alias of the
|
|
943
|
+
:param alias_or_id: Alias of the collection. If an existing ID is given, this
|
|
950
944
|
method will directly return the given value.
|
|
951
|
-
:returns: Internal name of the
|
|
945
|
+
:returns: Internal name of the collection.
|
|
952
946
|
"""
|
|
953
|
-
|
|
947
|
+
collections = [
|
|
954
948
|
k
|
|
955
|
-
for k, v in self.
|
|
949
|
+
for k, v in self.collections_config.items()
|
|
956
950
|
if v.get("alias") == alias_or_id
|
|
957
951
|
]
|
|
958
952
|
|
|
959
|
-
if len(
|
|
960
|
-
raise
|
|
961
|
-
f"Too many matching
|
|
953
|
+
if len(collections) > 1:
|
|
954
|
+
raise NoMatchingCollection(
|
|
955
|
+
f"Too many matching collections for alias {alias_or_id}: {collections}"
|
|
962
956
|
)
|
|
963
957
|
|
|
964
|
-
if len(
|
|
965
|
-
if alias_or_id in self.
|
|
958
|
+
if len(collections) == 0:
|
|
959
|
+
if alias_or_id in self.collections_config:
|
|
966
960
|
return alias_or_id
|
|
967
961
|
else:
|
|
968
|
-
raise
|
|
969
|
-
f"Could not find
|
|
962
|
+
raise NoMatchingCollection(
|
|
963
|
+
f"Could not find collection from alias or ID {alias_or_id}"
|
|
970
964
|
)
|
|
971
965
|
|
|
972
|
-
return
|
|
966
|
+
return collections[0]
|
|
973
967
|
|
|
974
|
-
def
|
|
975
|
-
"""Return the alias of a
|
|
976
|
-
given
|
|
968
|
+
def get_alias_from_collection(self, collection: str) -> str:
|
|
969
|
+
"""Return the alias of a collection by its ID. If no alias was defined for the
|
|
970
|
+
given collection, its ID is returned instead.
|
|
977
971
|
|
|
978
|
-
:param
|
|
979
|
-
:returns: Alias of the
|
|
972
|
+
:param collection: collection ID
|
|
973
|
+
:returns: Alias of the collection or its ID if no alias has been defined for it.
|
|
980
974
|
"""
|
|
981
|
-
if
|
|
982
|
-
raise
|
|
975
|
+
if collection not in self.collections_config:
|
|
976
|
+
raise NoMatchingCollection(collection)
|
|
983
977
|
|
|
984
|
-
return self.
|
|
978
|
+
return self.collections_config[collection].get("alias", collection)
|
|
985
979
|
|
|
986
|
-
def
|
|
980
|
+
def guess_collection(
|
|
987
981
|
self,
|
|
988
982
|
free_text: Optional[str] = None,
|
|
989
983
|
intersect: bool = False,
|
|
990
|
-
|
|
984
|
+
instruments: Optional[str] = None,
|
|
991
985
|
platform: Optional[str] = None,
|
|
992
|
-
|
|
993
|
-
|
|
994
|
-
|
|
986
|
+
constellation: Optional[str] = None,
|
|
987
|
+
processing_level: Optional[str] = None,
|
|
988
|
+
sensor_type: Optional[str] = None,
|
|
995
989
|
keywords: Optional[str] = None,
|
|
996
|
-
|
|
990
|
+
description: Optional[str] = None,
|
|
997
991
|
title: Optional[str] = None,
|
|
998
|
-
|
|
999
|
-
|
|
992
|
+
start_date: Optional[str] = None,
|
|
993
|
+
end_date: Optional[str] = None,
|
|
1000
994
|
**kwargs: Any,
|
|
1001
995
|
) -> list[str]:
|
|
1002
996
|
"""
|
|
1003
|
-
Find EODAG
|
|
997
|
+
Find EODAG collection IDs that best match a set of search parameters.
|
|
1004
998
|
|
|
1005
|
-
When using several filters,
|
|
999
|
+
When using several filters, collections that match most of them will be returned at first.
|
|
1006
1000
|
|
|
1007
1001
|
:param free_text: Free text search filter used to search accross all the following parameters. Handles logical
|
|
1008
1002
|
operators with parenthesis (``AND``/``OR``/``NOT``), quoted phrases (``"exact phrase"``),
|
|
@@ -1010,30 +1004,30 @@ class EODataAccessGateway:
|
|
|
1010
1004
|
:param intersect: Join results for each parameter using INTERSECT instead of UNION.
|
|
1011
1005
|
:param instrument: Instrument parameter.
|
|
1012
1006
|
:param platform: Platform parameter.
|
|
1013
|
-
:param
|
|
1014
|
-
:param
|
|
1015
|
-
:param
|
|
1007
|
+
:param constellation: Constellation parameter.
|
|
1008
|
+
:param processing_level: Processing level parameter.
|
|
1009
|
+
:param sensor_type: Sensor type parameter.
|
|
1016
1010
|
:param keywords: Keywords parameter.
|
|
1017
|
-
:param
|
|
1011
|
+
:param description: description parameter.
|
|
1018
1012
|
:param title: Title parameter.
|
|
1019
|
-
:param
|
|
1020
|
-
:param
|
|
1013
|
+
:param start_date: start date for datetime filtering. Not used by free_text
|
|
1014
|
+
:param end_date: end date for datetime filtering. Not used by free_text
|
|
1021
1015
|
:returns: The best match for the given parameters.
|
|
1022
|
-
:raises: :class:`~eodag.utils.exceptions.
|
|
1016
|
+
:raises: :class:`~eodag.utils.exceptions.NoMatchingCollection`
|
|
1023
1017
|
"""
|
|
1024
|
-
if
|
|
1025
|
-
return [
|
|
1018
|
+
if collection := kwargs.get("collection"):
|
|
1019
|
+
return [collection]
|
|
1026
1020
|
|
|
1027
1021
|
filters: dict[str, str] = {
|
|
1028
1022
|
k: v
|
|
1029
1023
|
for k, v in {
|
|
1030
|
-
"
|
|
1024
|
+
"instruments": instruments,
|
|
1025
|
+
"constellation": constellation,
|
|
1031
1026
|
"platform": platform,
|
|
1032
|
-
"
|
|
1033
|
-
"
|
|
1034
|
-
"sensorType": sensorType,
|
|
1027
|
+
"processing:level": processing_level,
|
|
1028
|
+
"eodag:sensor_type": sensor_type,
|
|
1035
1029
|
"keywords": keywords,
|
|
1036
|
-
"
|
|
1030
|
+
"description": description,
|
|
1037
1031
|
"title": title,
|
|
1038
1032
|
}.items()
|
|
1039
1033
|
if v is not None
|
|
@@ -1041,7 +1035,7 @@ class EODataAccessGateway:
|
|
|
1041
1035
|
|
|
1042
1036
|
only_dates = (
|
|
1043
1037
|
True
|
|
1044
|
-
if (not free_text and not filters and (
|
|
1038
|
+
if (not free_text and not filters and (start_date or end_date))
|
|
1045
1039
|
else False
|
|
1046
1040
|
)
|
|
1047
1041
|
|
|
@@ -1051,11 +1045,10 @@ class EODataAccessGateway:
|
|
|
1051
1045
|
|
|
1052
1046
|
guesses_with_score: list[tuple[str, int]] = []
|
|
1053
1047
|
|
|
1054
|
-
for pt_id, pt_dict in self.
|
|
1048
|
+
for pt_id, pt_dict in self.collections_config.source.items():
|
|
1055
1049
|
if (
|
|
1056
|
-
pt_id ==
|
|
1057
|
-
or pt_id
|
|
1058
|
-
not in self._plugins_manager.product_type_to_provider_config_map
|
|
1050
|
+
pt_id == GENERIC_COLLECTION
|
|
1051
|
+
or pt_id not in self._plugins_manager.collection_to_provider_config_map
|
|
1059
1052
|
):
|
|
1060
1053
|
continue
|
|
1061
1054
|
|
|
@@ -1094,25 +1087,19 @@ class EODataAccessGateway:
|
|
|
1094
1087
|
continue
|
|
1095
1088
|
|
|
1096
1089
|
# datetime filtering
|
|
1097
|
-
if
|
|
1090
|
+
if start_date or end_date:
|
|
1098
1091
|
min_aware = datetime.datetime.min.replace(tzinfo=datetime.timezone.utc)
|
|
1099
1092
|
max_aware = datetime.datetime.max.replace(tzinfo=datetime.timezone.utc)
|
|
1100
1093
|
|
|
1094
|
+
col_start, col_end = get_collection_dates(pt_dict)
|
|
1095
|
+
|
|
1101
1096
|
max_start = max(
|
|
1102
|
-
rfc3339_str_to_datetime(
|
|
1103
|
-
if
|
|
1104
|
-
else min_aware,
|
|
1105
|
-
rfc3339_str_to_datetime(pt_dict["missionStartDate"])
|
|
1106
|
-
if pt_dict.get("missionStartDate")
|
|
1107
|
-
else min_aware,
|
|
1097
|
+
rfc3339_str_to_datetime(start_date) if start_date else min_aware,
|
|
1098
|
+
rfc3339_str_to_datetime(col_start) if col_start else min_aware,
|
|
1108
1099
|
)
|
|
1109
1100
|
min_end = min(
|
|
1110
|
-
rfc3339_str_to_datetime(
|
|
1111
|
-
if
|
|
1112
|
-
else max_aware,
|
|
1113
|
-
rfc3339_str_to_datetime(pt_dict["missionEndDate"])
|
|
1114
|
-
if pt_dict.get("missionEndDate")
|
|
1115
|
-
else max_aware,
|
|
1101
|
+
rfc3339_str_to_datetime(end_date) if end_date else max_aware,
|
|
1102
|
+
rfc3339_str_to_datetime(col_end) if col_end else max_aware,
|
|
1116
1103
|
)
|
|
1117
1104
|
if not (max_start <= min_end):
|
|
1118
1105
|
continue
|
|
@@ -1125,7 +1112,7 @@ class EODataAccessGateway:
|
|
|
1125
1112
|
guesses_with_score.sort(key=lambda x: (-x[1], x[0]))
|
|
1126
1113
|
return [pt_id for pt_id, _ in guesses_with_score]
|
|
1127
1114
|
|
|
1128
|
-
raise
|
|
1115
|
+
raise NoMatchingCollection()
|
|
1129
1116
|
|
|
1130
1117
|
def search(
|
|
1131
1118
|
self,
|
|
@@ -1138,12 +1125,13 @@ class EODataAccessGateway:
|
|
|
1138
1125
|
locations: Optional[dict[str, str]] = None,
|
|
1139
1126
|
provider: Optional[str] = None,
|
|
1140
1127
|
count: bool = False,
|
|
1128
|
+
validate: Optional[bool] = True,
|
|
1141
1129
|
**kwargs: Any,
|
|
1142
1130
|
) -> SearchResult:
|
|
1143
1131
|
"""Look for products matching criteria on known providers.
|
|
1144
1132
|
|
|
1145
1133
|
The default behaviour is to look for products on the provider with the
|
|
1146
|
-
highest priority supporting the requested
|
|
1134
|
+
highest priority supporting the requested collection. These priorities
|
|
1147
1135
|
are configurable through user configuration file or individual environment variable.
|
|
1148
1136
|
If the request to the provider with the highest priority fails or is empty, the data
|
|
1149
1137
|
will be request from the provider with the next highest priority.
|
|
@@ -1178,9 +1166,11 @@ class EODataAccessGateway:
|
|
|
1178
1166
|
If not set, the configured preferred provider will be used at first
|
|
1179
1167
|
before trying others until finding results.
|
|
1180
1168
|
:param count: (optional) Whether to run a query with a count request or not
|
|
1169
|
+
:param validate: (optional) Set to True to validate search parameters
|
|
1170
|
+
before sending the query to the provider
|
|
1181
1171
|
:param kwargs: Some other criteria that will be used to do the search,
|
|
1182
1172
|
using paramaters compatibles with the provider
|
|
1183
|
-
:returns: A
|
|
1173
|
+
:returns: A set of EO products matching the criteria
|
|
1184
1174
|
|
|
1185
1175
|
.. versionchanged:: v3.0.0b1
|
|
1186
1176
|
``search()`` method now returns only a single :class:`~eodag.api.search_result.SearchResult`
|
|
@@ -1200,10 +1190,12 @@ class EODataAccessGateway:
|
|
|
1200
1190
|
**kwargs,
|
|
1201
1191
|
)
|
|
1202
1192
|
if search_kwargs.get("id"):
|
|
1193
|
+
# Don't validate requests by ID. "id" is not queryable.
|
|
1203
1194
|
return self._search_by_id(
|
|
1204
1195
|
search_kwargs.pop("id"),
|
|
1205
1196
|
provider=provider,
|
|
1206
1197
|
raise_errors=raise_errors,
|
|
1198
|
+
validate=False,
|
|
1207
1199
|
**search_kwargs,
|
|
1208
1200
|
)
|
|
1209
1201
|
# remove datacube query string from kwargs which was only needed for search-by-id
|
|
@@ -1222,6 +1214,7 @@ class EODataAccessGateway:
|
|
|
1222
1214
|
search_plugin,
|
|
1223
1215
|
count=count,
|
|
1224
1216
|
raise_errors=raise_errors,
|
|
1217
|
+
validate=validate,
|
|
1225
1218
|
**search_kwargs,
|
|
1226
1219
|
)
|
|
1227
1220
|
errors.extend(search_results.errors)
|
|
@@ -1272,7 +1265,7 @@ class EODataAccessGateway:
|
|
|
1272
1265
|
name=country and attr=ISO3
|
|
1273
1266
|
:param kwargs: Some other criteria that will be used to do the search,
|
|
1274
1267
|
using paramaters compatibles with the provider
|
|
1275
|
-
:returns: An iterator that yields page per page a
|
|
1268
|
+
:returns: An iterator that yields page per page a set of EO products
|
|
1276
1269
|
matching the criteria
|
|
1277
1270
|
"""
|
|
1278
1271
|
search_plugins, search_kwargs = self._prepare_search(
|
|
@@ -1311,7 +1304,7 @@ class EODataAccessGateway:
|
|
|
1311
1304
|
:param kwargs: Some other criteria that will be used to do the search,
|
|
1312
1305
|
using parameters compatibles with the provider
|
|
1313
1306
|
:param search_plugin: search plugin to be used
|
|
1314
|
-
:returns: An iterator that yields page per page a
|
|
1307
|
+
:returns: An iterator that yields page per page a set of EO products
|
|
1315
1308
|
matching the criteria
|
|
1316
1309
|
"""
|
|
1317
1310
|
|
|
@@ -1471,27 +1464,27 @@ class EODataAccessGateway:
|
|
|
1471
1464
|
name=country and attr=ISO3
|
|
1472
1465
|
:param kwargs: Some other criteria that will be used to do the search,
|
|
1473
1466
|
using parameters compatible with the provider
|
|
1474
|
-
:returns: An iterator that yields page per page a
|
|
1467
|
+
:returns: An iterator that yields page per page a set of EO products
|
|
1475
1468
|
matching the criteria
|
|
1476
1469
|
"""
|
|
1477
1470
|
# Get the search plugin and the maximized value
|
|
1478
1471
|
# of items_per_page if defined for the provider used.
|
|
1479
1472
|
try:
|
|
1480
|
-
|
|
1481
|
-
self.
|
|
1473
|
+
collection = self.get_collection_from_alias(
|
|
1474
|
+
self.guess_collection(**kwargs)[0]
|
|
1482
1475
|
)
|
|
1483
|
-
except
|
|
1484
|
-
|
|
1476
|
+
except NoMatchingCollection:
|
|
1477
|
+
collection = GENERIC_COLLECTION
|
|
1485
1478
|
else:
|
|
1486
|
-
# fetch
|
|
1479
|
+
# fetch collections list if collection is unknown
|
|
1487
1480
|
if (
|
|
1488
|
-
|
|
1489
|
-
not in self._plugins_manager.
|
|
1481
|
+
collection
|
|
1482
|
+
not in self._plugins_manager.collection_to_provider_config_map.keys()
|
|
1490
1483
|
):
|
|
1491
1484
|
logger.debug(
|
|
1492
|
-
f"Fetching external
|
|
1485
|
+
f"Fetching external collections sources to find {collection} collection"
|
|
1493
1486
|
)
|
|
1494
|
-
self.
|
|
1487
|
+
self.fetch_collections_list()
|
|
1495
1488
|
|
|
1496
1489
|
# remove unwanted count
|
|
1497
1490
|
kwargs.pop("count", None)
|
|
@@ -1571,13 +1564,13 @@ class EODataAccessGateway:
|
|
|
1571
1564
|
:param kwargs: Search criteria to help finding the right product
|
|
1572
1565
|
:returns: A search result with one EO product or None at all
|
|
1573
1566
|
"""
|
|
1574
|
-
|
|
1575
|
-
if
|
|
1567
|
+
collection = kwargs.get("collection")
|
|
1568
|
+
if collection is not None:
|
|
1576
1569
|
try:
|
|
1577
|
-
|
|
1578
|
-
except
|
|
1579
|
-
logger.debug("
|
|
1580
|
-
get_search_plugins_kwargs = dict(provider=provider,
|
|
1570
|
+
collection = self.get_collection_from_alias(collection)
|
|
1571
|
+
except NoMatchingCollection:
|
|
1572
|
+
logger.debug("collection %s not found", collection)
|
|
1573
|
+
get_search_plugins_kwargs = dict(provider=provider, collection=collection)
|
|
1581
1574
|
search_plugins = self._plugins_manager.get_search_plugins(
|
|
1582
1575
|
**get_search_plugins_kwargs
|
|
1583
1576
|
)
|
|
@@ -1632,10 +1625,10 @@ class EODataAccessGateway:
|
|
|
1632
1625
|
results = filtered
|
|
1633
1626
|
|
|
1634
1627
|
if len(results) == 1:
|
|
1635
|
-
if not results[0].
|
|
1636
|
-
# guess
|
|
1637
|
-
guesses = self.
|
|
1638
|
-
results[0].
|
|
1628
|
+
if not results[0].collection:
|
|
1629
|
+
# guess collection from properties
|
|
1630
|
+
guesses = self.guess_collection(**results[0].properties)
|
|
1631
|
+
results[0].collection = guesses[0]
|
|
1639
1632
|
# reset driver
|
|
1640
1633
|
results[0].driver = results[0].get_driver()
|
|
1641
1634
|
results.number_matched = 1
|
|
@@ -1647,15 +1640,15 @@ class EODataAccessGateway:
|
|
|
1647
1640
|
)
|
|
1648
1641
|
return SearchResult([], 0, results.errors)
|
|
1649
1642
|
|
|
1650
|
-
def
|
|
1643
|
+
def _fetch_external_collection(self, provider: str, collection: str):
|
|
1651
1644
|
plugins = self._plugins_manager.get_search_plugins(provider=provider)
|
|
1652
1645
|
plugin = next(plugins)
|
|
1653
1646
|
|
|
1654
1647
|
# check after plugin init if still fetchable
|
|
1655
|
-
if not getattr(plugin.config, "
|
|
1648
|
+
if not getattr(plugin.config, "discover_collections", {}).get("fetch_url"):
|
|
1656
1649
|
return None
|
|
1657
1650
|
|
|
1658
|
-
kwargs: dict[str, Any] = {"
|
|
1651
|
+
kwargs: dict[str, Any] = {"collection": collection}
|
|
1659
1652
|
|
|
1660
1653
|
# append auth if needed
|
|
1661
1654
|
if getattr(plugin.config, "need_auth", False):
|
|
@@ -1666,8 +1659,8 @@ class EODataAccessGateway:
|
|
|
1666
1659
|
):
|
|
1667
1660
|
kwargs["auth"] = auth
|
|
1668
1661
|
|
|
1669
|
-
|
|
1670
|
-
self.
|
|
1662
|
+
collection_config = plugin.discover_collections(**kwargs)
|
|
1663
|
+
self.update_collections_list({provider: collection_config})
|
|
1671
1664
|
|
|
1672
1665
|
def _prepare_search(
|
|
1673
1666
|
self,
|
|
@@ -1683,9 +1676,9 @@ class EODataAccessGateway:
|
|
|
1683
1676
|
Product query:
|
|
1684
1677
|
* By id (plus optional 'provider')
|
|
1685
1678
|
* By search params:
|
|
1686
|
-
*
|
|
1687
|
-
* By
|
|
1688
|
-
* By params (e.g. 'platform'), see
|
|
1679
|
+
* collection query:
|
|
1680
|
+
* By collection (e.g. 'S2_MSI_L1C')
|
|
1681
|
+
* By params (e.g. 'platform'), see guess_collection
|
|
1689
1682
|
* dates: 'start' and/or 'end'
|
|
1690
1683
|
* geometry: 'geom' or 'bbox' or 'box'
|
|
1691
1684
|
* search locations
|
|
@@ -1700,53 +1693,53 @@ class EODataAccessGateway:
|
|
|
1700
1693
|
If no time offset is given, the time is assumed to be given in UTC.
|
|
1701
1694
|
:param geom: (optional) Search area that can be defined in different ways (see search)
|
|
1702
1695
|
:param locations: (optional) Location filtering by name using locations configuration
|
|
1703
|
-
:param provider: provider to be used, if no provider is given or the
|
|
1696
|
+
:param provider: provider to be used, if no provider is given or the collection
|
|
1704
1697
|
is not available for the provider, the preferred provider is used
|
|
1705
1698
|
:param kwargs: Some other criteria
|
|
1706
1699
|
* id and/or a provider for a search by
|
|
1707
|
-
* search criteria to guess the
|
|
1700
|
+
* search criteria to guess the collection
|
|
1708
1701
|
* other criteria compatible with the provider
|
|
1709
1702
|
:returns: Search plugins list and the prepared kwargs to make a query.
|
|
1710
1703
|
"""
|
|
1711
|
-
|
|
1712
|
-
if
|
|
1704
|
+
collection: Optional[str] = kwargs.get("collection")
|
|
1705
|
+
if collection is None:
|
|
1713
1706
|
try:
|
|
1714
|
-
guesses = self.
|
|
1707
|
+
guesses = self.guess_collection(**kwargs)
|
|
1715
1708
|
|
|
1716
|
-
#
|
|
1709
|
+
# guess_collection raises a NoMatchingCollection error if no product
|
|
1717
1710
|
# is found. Here, the supported search params are removed from the
|
|
1718
1711
|
# kwargs if present, not to propagate them to the query itself.
|
|
1719
1712
|
for param in (
|
|
1720
|
-
"
|
|
1713
|
+
"instruments",
|
|
1714
|
+
"constellation",
|
|
1721
1715
|
"platform",
|
|
1722
|
-
"
|
|
1723
|
-
"
|
|
1724
|
-
"sensorType",
|
|
1716
|
+
"processing:level",
|
|
1717
|
+
"eodag:sensor_type",
|
|
1725
1718
|
):
|
|
1726
1719
|
kwargs.pop(param, None)
|
|
1727
1720
|
|
|
1728
1721
|
# By now, only use the best bet
|
|
1729
|
-
|
|
1730
|
-
except
|
|
1722
|
+
collection = guesses[0]
|
|
1723
|
+
except NoMatchingCollection:
|
|
1731
1724
|
queried_id = kwargs.get("id")
|
|
1732
1725
|
if queried_id is None:
|
|
1733
1726
|
logger.info(
|
|
1734
|
-
"No
|
|
1727
|
+
"No collection could be guessed with provided arguments"
|
|
1735
1728
|
)
|
|
1736
1729
|
else:
|
|
1737
1730
|
return [], kwargs
|
|
1738
1731
|
|
|
1739
|
-
if
|
|
1732
|
+
if collection is not None:
|
|
1740
1733
|
try:
|
|
1741
|
-
|
|
1742
|
-
except
|
|
1743
|
-
logger.info("unknown
|
|
1744
|
-
kwargs["
|
|
1734
|
+
collection = self.get_collection_from_alias(collection)
|
|
1735
|
+
except NoMatchingCollection:
|
|
1736
|
+
logger.info("unknown collection " + collection)
|
|
1737
|
+
kwargs["collection"] = collection
|
|
1745
1738
|
|
|
1746
1739
|
if start is not None:
|
|
1747
|
-
kwargs["
|
|
1740
|
+
kwargs["start_datetime"] = start
|
|
1748
1741
|
if end is not None:
|
|
1749
|
-
kwargs["
|
|
1742
|
+
kwargs["end_datetime"] = end
|
|
1750
1743
|
if "box" in kwargs or "bbox" in kwargs:
|
|
1751
1744
|
logger.warning(
|
|
1752
1745
|
"'box' or 'bbox' parameters are only supported for backwards "
|
|
@@ -1767,33 +1760,42 @@ class EODataAccessGateway:
|
|
|
1767
1760
|
kwargs.pop(arg, None)
|
|
1768
1761
|
del kwargs["locations"]
|
|
1769
1762
|
|
|
1770
|
-
# fetch
|
|
1763
|
+
# fetch collections list if collection is unknown
|
|
1771
1764
|
if (
|
|
1772
|
-
|
|
1773
|
-
not in self._plugins_manager.
|
|
1765
|
+
collection
|
|
1766
|
+
not in self._plugins_manager.collection_to_provider_config_map.keys()
|
|
1774
1767
|
):
|
|
1775
|
-
if provider and
|
|
1776
|
-
#
|
|
1777
|
-
logger.debug(
|
|
1778
|
-
|
|
1768
|
+
if provider and collection:
|
|
1769
|
+
# fetch ref for given provider and collection
|
|
1770
|
+
logger.debug(
|
|
1771
|
+
f"Fetching external collections sources to find {provider} {collection} collection"
|
|
1772
|
+
)
|
|
1773
|
+
self.fetch_collections_list(provider)
|
|
1774
|
+
if (
|
|
1775
|
+
collection
|
|
1776
|
+
not in self._plugins_manager.collection_to_provider_config_map.keys()
|
|
1777
|
+
):
|
|
1778
|
+
# Try to get specific collection from external provider
|
|
1779
|
+
logger.debug(f"Fetching {provider} to find {collection} collection")
|
|
1780
|
+
self._fetch_external_collection(provider, collection)
|
|
1779
1781
|
if not provider:
|
|
1780
|
-
# no provider or still not found -> fetch all external
|
|
1782
|
+
# no provider or still not found -> fetch all external collections
|
|
1781
1783
|
logger.debug(
|
|
1782
|
-
f"Fetching external
|
|
1784
|
+
f"Fetching external collections sources to find {collection} collection"
|
|
1783
1785
|
)
|
|
1784
|
-
self.
|
|
1786
|
+
self.fetch_collections_list()
|
|
1785
1787
|
|
|
1786
1788
|
preferred_provider = self.get_preferred_provider()[0]
|
|
1787
1789
|
|
|
1788
1790
|
search_plugins: list[Union[Search, Api]] = []
|
|
1789
1791
|
for plugin in self._plugins_manager.get_search_plugins(
|
|
1790
|
-
|
|
1792
|
+
collection=collection, provider=provider
|
|
1791
1793
|
):
|
|
1792
|
-
# exclude MeteoblueSearch plugins from search fallback for unknown
|
|
1794
|
+
# exclude MeteoblueSearch plugins from search fallback for unknown collection
|
|
1793
1795
|
if (
|
|
1794
1796
|
provider != plugin.provider
|
|
1795
1797
|
and preferred_provider != plugin.provider
|
|
1796
|
-
and
|
|
1798
|
+
and collection not in self.collections_config
|
|
1797
1799
|
and isinstance(plugin, MeteoblueSearch)
|
|
1798
1800
|
):
|
|
1799
1801
|
continue
|
|
@@ -1804,8 +1806,8 @@ class EODataAccessGateway:
|
|
|
1804
1806
|
providers = [plugin.provider for plugin in search_plugins]
|
|
1805
1807
|
if provider not in providers:
|
|
1806
1808
|
logger.debug(
|
|
1807
|
-
"
|
|
1808
|
-
|
|
1809
|
+
"Collection '%s' is not available with preferred provider '%s'.",
|
|
1810
|
+
collection,
|
|
1809
1811
|
provider,
|
|
1810
1812
|
)
|
|
1811
1813
|
else:
|
|
@@ -1814,11 +1816,11 @@ class EODataAccessGateway:
|
|
|
1814
1816
|
)[0]
|
|
1815
1817
|
search_plugins.remove(provider_plugin)
|
|
1816
1818
|
search_plugins.insert(0, provider_plugin)
|
|
1817
|
-
# Add
|
|
1819
|
+
# Add collections_config to plugin config. This dict contains product
|
|
1818
1820
|
# type metadata that will also be stored in each product's properties.
|
|
1819
1821
|
for search_plugin in search_plugins:
|
|
1820
|
-
if
|
|
1821
|
-
self.
|
|
1822
|
+
if collection is not None:
|
|
1823
|
+
self._attach_collection_config(search_plugin, collection)
|
|
1822
1824
|
|
|
1823
1825
|
return search_plugins, kwargs
|
|
1824
1826
|
|
|
@@ -1827,6 +1829,7 @@ class EODataAccessGateway:
|
|
|
1827
1829
|
search_plugin: Union[Search, Api],
|
|
1828
1830
|
count: bool = False,
|
|
1829
1831
|
raise_errors: bool = False,
|
|
1832
|
+
validate: Optional[bool] = True,
|
|
1830
1833
|
**kwargs: Any,
|
|
1831
1834
|
) -> SearchResult:
|
|
1832
1835
|
"""Internal method that performs a search on a given provider.
|
|
@@ -1836,6 +1839,8 @@ class EODataAccessGateway:
|
|
|
1836
1839
|
:param raise_errors: (optional) When an error occurs when searching, if this is set to
|
|
1837
1840
|
True, the error is raised
|
|
1838
1841
|
:param kwargs: Some other criteria that will be used to do the search
|
|
1842
|
+
:param validate: (optional) Set to True to validate search parameters
|
|
1843
|
+
before sending the query to the provider
|
|
1839
1844
|
:returns: A collection of EO products matching the criteria
|
|
1840
1845
|
"""
|
|
1841
1846
|
logger.info("Searching on provider %s", search_plugin.provider)
|
|
@@ -1876,7 +1881,24 @@ class EODataAccessGateway:
|
|
|
1876
1881
|
prep.page = kwargs.pop("page", None)
|
|
1877
1882
|
prep.items_per_page = kwargs.pop("items_per_page", None)
|
|
1878
1883
|
|
|
1879
|
-
|
|
1884
|
+
# remove None values and convert param names to their pydantic alias if any
|
|
1885
|
+
search_params = {}
|
|
1886
|
+
for param, value in kwargs.items():
|
|
1887
|
+
if value is None:
|
|
1888
|
+
continue
|
|
1889
|
+
if param in Queryables.model_fields:
|
|
1890
|
+
param_alias = Queryables.model_fields[param].alias or param
|
|
1891
|
+
search_params[param_alias] = value
|
|
1892
|
+
else:
|
|
1893
|
+
# remove `provider:` or `provider_` prefix if any
|
|
1894
|
+
search_params[
|
|
1895
|
+
re.sub(r"^" + search_plugin.provider + r"[_:]", "", param)
|
|
1896
|
+
] = value
|
|
1897
|
+
|
|
1898
|
+
if validate:
|
|
1899
|
+
search_plugin.validate(search_params, prep.auth)
|
|
1900
|
+
|
|
1901
|
+
res, nb_res = search_plugin.query(prep, **search_params)
|
|
1880
1902
|
|
|
1881
1903
|
if not isinstance(res, list):
|
|
1882
1904
|
raise PluginImplementationError(
|
|
@@ -1894,39 +1916,39 @@ class EODataAccessGateway:
|
|
|
1894
1916
|
# be returned as a search result if there was no search extent (because we
|
|
1895
1917
|
# will not try to do an intersection)
|
|
1896
1918
|
for eo_product in res:
|
|
1897
|
-
# if
|
|
1898
|
-
if eo_product.
|
|
1919
|
+
# if collection is not defined, try to guess using properties
|
|
1920
|
+
if eo_product.collection is None:
|
|
1899
1921
|
pattern = re.compile(r"[^\w,]+")
|
|
1900
1922
|
try:
|
|
1901
|
-
guesses = self.
|
|
1923
|
+
guesses = self.guess_collection(
|
|
1902
1924
|
intersect=False,
|
|
1903
1925
|
**{
|
|
1904
1926
|
k: pattern.sub("", str(v).upper())
|
|
1905
1927
|
for k, v in eo_product.properties.items()
|
|
1906
1928
|
if k
|
|
1907
1929
|
in [
|
|
1908
|
-
"
|
|
1930
|
+
"instruments",
|
|
1931
|
+
"constellation",
|
|
1909
1932
|
"platform",
|
|
1910
|
-
"
|
|
1911
|
-
"
|
|
1912
|
-
"sensorType",
|
|
1933
|
+
"processing:level",
|
|
1934
|
+
"eodag:sensor_type",
|
|
1913
1935
|
"keywords",
|
|
1914
1936
|
]
|
|
1915
1937
|
and v is not None
|
|
1916
1938
|
},
|
|
1917
1939
|
)
|
|
1918
|
-
except
|
|
1940
|
+
except NoMatchingCollection:
|
|
1919
1941
|
pass
|
|
1920
1942
|
else:
|
|
1921
|
-
eo_product.
|
|
1943
|
+
eo_product.collection = guesses[0]
|
|
1922
1944
|
|
|
1923
1945
|
try:
|
|
1924
|
-
if eo_product.
|
|
1925
|
-
eo_product.
|
|
1926
|
-
eo_product.
|
|
1946
|
+
if eo_product.collection is not None:
|
|
1947
|
+
eo_product.collection = self.get_collection_from_alias(
|
|
1948
|
+
eo_product.collection
|
|
1927
1949
|
)
|
|
1928
|
-
except
|
|
1929
|
-
logger.debug("
|
|
1950
|
+
except NoMatchingCollection:
|
|
1951
|
+
logger.debug("collection %s not found", eo_product.collection)
|
|
1930
1952
|
|
|
1931
1953
|
if eo_product.search_intersection is not None:
|
|
1932
1954
|
eo_product._register_downloader_from_manager(self._plugins_manager)
|
|
@@ -2005,7 +2027,7 @@ class EODataAccessGateway:
|
|
|
2005
2027
|
) -> list[str]:
|
|
2006
2028
|
"""Download all products resulting from a search.
|
|
2007
2029
|
|
|
2008
|
-
:param search_result: A
|
|
2030
|
+
:param search_result: A set of EO products resulting from a search
|
|
2009
2031
|
:param downloaded_callback: (optional) A method or a callable object which takes
|
|
2010
2032
|
as parameter the ``product``. You can use the base class
|
|
2011
2033
|
:class:`~eodag.utils.DownloadedCallback` and override
|
|
@@ -2059,7 +2081,7 @@ class EODataAccessGateway:
|
|
|
2059
2081
|
) -> str:
|
|
2060
2082
|
"""Registers results of a search into a geojson file.
|
|
2061
2083
|
|
|
2062
|
-
:param search_result: A
|
|
2084
|
+
:param search_result: A set of EO products resulting from a search
|
|
2063
2085
|
:param filename: (optional) The name of the file to generate
|
|
2064
2086
|
:returns: The name of the created file
|
|
2065
2087
|
"""
|
|
@@ -2095,69 +2117,6 @@ class EODataAccessGateway:
|
|
|
2095
2117
|
|
|
2096
2118
|
return products
|
|
2097
2119
|
|
|
2098
|
-
@_deprecated(
|
|
2099
|
-
reason="Use the StaticStacSearch search plugin instead", version="2.2.1"
|
|
2100
|
-
)
|
|
2101
|
-
def load_stac_items(
|
|
2102
|
-
self,
|
|
2103
|
-
filename: str,
|
|
2104
|
-
recursive: bool = False,
|
|
2105
|
-
max_connections: int = 100,
|
|
2106
|
-
provider: Optional[str] = None,
|
|
2107
|
-
productType: Optional[str] = None,
|
|
2108
|
-
timeout: int = HTTP_REQ_TIMEOUT,
|
|
2109
|
-
ssl_verify: bool = True,
|
|
2110
|
-
**kwargs: Any,
|
|
2111
|
-
) -> SearchResult:
|
|
2112
|
-
"""Loads STAC items from a geojson file / STAC catalog or collection, and convert to SearchResult.
|
|
2113
|
-
|
|
2114
|
-
Features are parsed using eodag provider configuration, as if they were
|
|
2115
|
-
the response content to an API request.
|
|
2116
|
-
|
|
2117
|
-
:param filename: A filename containing features encoded as a geojson
|
|
2118
|
-
:param recursive: (optional) Browse recursively in child nodes if True
|
|
2119
|
-
:param max_connections: (optional) Maximum number of connections for concurrent HTTP requests
|
|
2120
|
-
:param provider: (optional) Data provider
|
|
2121
|
-
:param productType: (optional) Data product type
|
|
2122
|
-
:param timeout: (optional) Timeout in seconds for each internal HTTP request
|
|
2123
|
-
:param kwargs: Parameters that will be stored in the result as
|
|
2124
|
-
search criteria
|
|
2125
|
-
:returns: The search results encoded in `filename`
|
|
2126
|
-
|
|
2127
|
-
.. deprecated:: 2.2.1
|
|
2128
|
-
Use the :class:`~eodag.plugins.search.static_stac_search.StaticStacSearch` search plugin instead.
|
|
2129
|
-
"""
|
|
2130
|
-
features = fetch_stac_items(
|
|
2131
|
-
filename,
|
|
2132
|
-
recursive=recursive,
|
|
2133
|
-
max_connections=max_connections,
|
|
2134
|
-
timeout=timeout,
|
|
2135
|
-
ssl_verify=ssl_verify,
|
|
2136
|
-
)
|
|
2137
|
-
feature_collection = geojson.FeatureCollection(features)
|
|
2138
|
-
|
|
2139
|
-
plugin = next(
|
|
2140
|
-
self._plugins_manager.get_search_plugins(
|
|
2141
|
-
product_type=productType, provider=provider
|
|
2142
|
-
)
|
|
2143
|
-
)
|
|
2144
|
-
# save plugin._request and mock it to make return loaded static results
|
|
2145
|
-
plugin_request = plugin._request
|
|
2146
|
-
plugin._request = (
|
|
2147
|
-
lambda url, info_message=None, exception_message=None: MockResponse(
|
|
2148
|
-
feature_collection, 200
|
|
2149
|
-
)
|
|
2150
|
-
)
|
|
2151
|
-
|
|
2152
|
-
search_result = self.search(
|
|
2153
|
-
productType=productType, provider=provider, **kwargs
|
|
2154
|
-
)
|
|
2155
|
-
|
|
2156
|
-
# restore plugin._request
|
|
2157
|
-
plugin._request = plugin_request
|
|
2158
|
-
|
|
2159
|
-
return search_result
|
|
2160
|
-
|
|
2161
2120
|
def download(
|
|
2162
2121
|
self,
|
|
2163
2122
|
product: EOProduct,
|
|
@@ -2173,16 +2132,16 @@ class EODataAccessGateway:
|
|
|
2173
2132
|
checks like verifying that a downloader and authenticator are registered
|
|
2174
2133
|
for the product before trying to download it.
|
|
2175
2134
|
|
|
2176
|
-
If the metadata mapping for ``
|
|
2135
|
+
If the metadata mapping for ``eodag:download_link`` is set to something that can be
|
|
2177
2136
|
interpreted as a link on a
|
|
2178
2137
|
local filesystem, the download is skipped (by now, only a link starting
|
|
2179
2138
|
with ``file:/`` is supported). Therefore, any user that knows how to extract
|
|
2180
2139
|
product location from product metadata on a provider can override the
|
|
2181
|
-
``
|
|
2140
|
+
``eodag:download_link`` metadata mapping in the right way. For example, using the
|
|
2182
2141
|
environment variable:
|
|
2183
|
-
``
|
|
2142
|
+
``EODAG__CREODIAS__SEARCH__METADATA_MAPPING__EODAG_DOWNLOAD_LINK="file:///{id}"`` will
|
|
2184
2143
|
lead to all :class:`~eodag.api.product._product.EOProduct`'s originating from the
|
|
2185
|
-
provider ``creodias`` to have their ``
|
|
2144
|
+
provider ``creodias`` to have their ``eodag:download_link`` metadata point to something like:
|
|
2186
2145
|
``file:///12345-678``, making this method immediately return the later string without
|
|
2187
2146
|
trying to download the product.
|
|
2188
2147
|
|
|
@@ -2246,46 +2205,46 @@ class EODataAccessGateway:
|
|
|
2246
2205
|
fetch_providers: bool = True,
|
|
2247
2206
|
**kwargs: Any,
|
|
2248
2207
|
) -> QueryablesDict:
|
|
2249
|
-
"""Fetch the queryable properties for a given
|
|
2208
|
+
"""Fetch the queryable properties for a given collection and/or provider.
|
|
2250
2209
|
|
|
2251
2210
|
:param provider: (optional) The provider.
|
|
2252
|
-
:param fetch_providers: If new
|
|
2253
|
-
:param kwargs: additional filters for queryables (`
|
|
2211
|
+
:param fetch_providers: If new collections should be fetched from the providers; default: True
|
|
2212
|
+
:param kwargs: additional filters for queryables (`collection` or other search
|
|
2254
2213
|
arguments)
|
|
2255
2214
|
|
|
2256
|
-
:raises
|
|
2215
|
+
:raises UnsupportedCollection: If the specified collection is not available for the
|
|
2257
2216
|
provider.
|
|
2258
2217
|
|
|
2259
2218
|
:returns: A :class:`~eodag.api.product.queryables.QuerybalesDict` containing the EODAG queryable
|
|
2260
2219
|
properties, associating parameters to their annotated type, and a additional_properties attribute
|
|
2261
2220
|
"""
|
|
2262
|
-
# only fetch providers if
|
|
2263
|
-
|
|
2221
|
+
# only fetch providers if collection is not found
|
|
2222
|
+
available_collections: list[str] = [
|
|
2264
2223
|
pt["ID"]
|
|
2265
|
-
for pt in self.
|
|
2224
|
+
for pt in self.list_collections(provider=provider, fetch_providers=False)
|
|
2266
2225
|
]
|
|
2267
|
-
|
|
2268
|
-
|
|
2226
|
+
collection: Optional[str] = kwargs.get("collection")
|
|
2227
|
+
coll_alias: Optional[str] = collection
|
|
2269
2228
|
|
|
2270
|
-
if
|
|
2271
|
-
if
|
|
2229
|
+
if collection:
|
|
2230
|
+
if collection not in available_collections:
|
|
2272
2231
|
if fetch_providers:
|
|
2273
2232
|
# fetch providers and try again
|
|
2274
|
-
|
|
2233
|
+
available_collections = [
|
|
2275
2234
|
pt["ID"]
|
|
2276
|
-
for pt in self.
|
|
2235
|
+
for pt in self.list_collections(
|
|
2277
2236
|
provider=provider, fetch_providers=True
|
|
2278
2237
|
)
|
|
2279
2238
|
]
|
|
2280
|
-
raise
|
|
2239
|
+
raise UnsupportedCollection(f"{collection} is not available.")
|
|
2281
2240
|
try:
|
|
2282
|
-
kwargs["
|
|
2283
|
-
|
|
2241
|
+
kwargs["collection"] = collection = self.get_collection_from_alias(
|
|
2242
|
+
collection
|
|
2284
2243
|
)
|
|
2285
|
-
except
|
|
2286
|
-
raise
|
|
2244
|
+
except NoMatchingCollection as e:
|
|
2245
|
+
raise UnsupportedCollection(f"{collection} is not available.") from e
|
|
2287
2246
|
|
|
2288
|
-
if not provider and not
|
|
2247
|
+
if not provider and not collection:
|
|
2289
2248
|
return QueryablesDict(
|
|
2290
2249
|
additional_properties=True,
|
|
2291
2250
|
**model_fields_to_annotated(CommonQueryables.model_fields),
|
|
@@ -2295,16 +2254,16 @@ class EODataAccessGateway:
|
|
|
2295
2254
|
additional_information = []
|
|
2296
2255
|
queryable_properties: dict[str, Any] = {}
|
|
2297
2256
|
|
|
2298
|
-
for plugin in self._plugins_manager.get_search_plugins(
|
|
2299
|
-
# attach
|
|
2300
|
-
|
|
2301
|
-
if
|
|
2302
|
-
self.
|
|
2303
|
-
|
|
2257
|
+
for plugin in self._plugins_manager.get_search_plugins(collection, provider):
|
|
2258
|
+
# attach collection config
|
|
2259
|
+
collection_configs: dict[str, Any] = {}
|
|
2260
|
+
if collection:
|
|
2261
|
+
self._attach_collection_config(plugin, collection)
|
|
2262
|
+
collection_configs[collection] = plugin.config.collection_config
|
|
2304
2263
|
else:
|
|
2305
|
-
for pt in
|
|
2306
|
-
self.
|
|
2307
|
-
|
|
2264
|
+
for pt in available_collections:
|
|
2265
|
+
self._attach_collection_config(plugin, pt)
|
|
2266
|
+
collection_configs[pt] = plugin.config.collection_config
|
|
2308
2267
|
|
|
2309
2268
|
# authenticate if required
|
|
2310
2269
|
if getattr(plugin.config, "need_auth", False) and (
|
|
@@ -2320,10 +2279,10 @@ class EODataAccessGateway:
|
|
|
2320
2279
|
|
|
2321
2280
|
plugin_queryables = plugin.list_queryables(
|
|
2322
2281
|
kwargs,
|
|
2323
|
-
|
|
2324
|
-
|
|
2325
|
-
|
|
2326
|
-
|
|
2282
|
+
available_collections,
|
|
2283
|
+
collection_configs,
|
|
2284
|
+
collection,
|
|
2285
|
+
coll_alias,
|
|
2327
2286
|
)
|
|
2328
2287
|
|
|
2329
2288
|
if plugin_queryables.additional_information:
|
|
@@ -2373,32 +2332,32 @@ class EODataAccessGateway:
|
|
|
2373
2332
|
}
|
|
2374
2333
|
return sortables
|
|
2375
2334
|
|
|
2376
|
-
def
|
|
2335
|
+
def _attach_collection_config(self, plugin: Search, collection: str) -> None:
|
|
2377
2336
|
"""
|
|
2378
|
-
Attach
|
|
2337
|
+
Attach collections_config to plugin config. This dict contains product
|
|
2379
2338
|
type metadata that will also be stored in each product's properties.
|
|
2380
2339
|
"""
|
|
2381
2340
|
try:
|
|
2382
|
-
plugin.config.
|
|
2341
|
+
plugin.config.collection_config = dict(
|
|
2383
2342
|
[
|
|
2384
2343
|
p
|
|
2385
|
-
for p in self.
|
|
2344
|
+
for p in self.list_collections(
|
|
2386
2345
|
plugin.provider, fetch_providers=False
|
|
2387
2346
|
)
|
|
2388
|
-
if p["_id"] ==
|
|
2347
|
+
if p["_id"] == collection
|
|
2389
2348
|
][0],
|
|
2390
|
-
**{"
|
|
2349
|
+
**{"collection": collection},
|
|
2391
2350
|
)
|
|
2392
|
-
# If the product isn't in the catalog, it's a generic
|
|
2351
|
+
# If the product isn't in the catalog, it's a generic collection.
|
|
2393
2352
|
except IndexError:
|
|
2394
|
-
# Construct the
|
|
2395
|
-
plugin.config.
|
|
2396
|
-
ID=
|
|
2397
|
-
**self.
|
|
2398
|
-
|
|
2353
|
+
# Construct the GENERIC_COLLECTION metadata
|
|
2354
|
+
plugin.config.collection_config = dict(
|
|
2355
|
+
ID=GENERIC_COLLECTION,
|
|
2356
|
+
**self.collections_config[GENERIC_COLLECTION],
|
|
2357
|
+
collection=collection,
|
|
2399
2358
|
)
|
|
2400
|
-
# Remove the ID since this is equal to
|
|
2401
|
-
plugin.config.
|
|
2359
|
+
# Remove the ID since this is equal to collection.
|
|
2360
|
+
plugin.config.collection_config.pop("ID", None)
|
|
2402
2361
|
|
|
2403
2362
|
def import_stac_items(self, items_urls: list[str]) -> SearchResult:
|
|
2404
2363
|
"""Import STAC items from a list of URLs and convert them to SearchResult.
|