eodag 3.10.1__py3-none-any.whl → 4.0.0a2__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- eodag/__init__.py +6 -1
- eodag/api/collection.py +353 -0
- eodag/api/core.py +606 -641
- eodag/api/product/__init__.py +3 -3
- eodag/api/product/_product.py +74 -56
- eodag/api/product/drivers/__init__.py +4 -46
- eodag/api/product/drivers/base.py +0 -28
- eodag/api/product/metadata_mapping.py +178 -216
- eodag/api/search_result.py +156 -15
- eodag/cli.py +83 -403
- eodag/config.py +81 -51
- eodag/plugins/apis/base.py +2 -2
- eodag/plugins/apis/ecmwf.py +36 -25
- eodag/plugins/apis/usgs.py +55 -40
- eodag/plugins/authentication/base.py +1 -3
- eodag/plugins/crunch/filter_date.py +3 -3
- eodag/plugins/crunch/filter_latest_intersect.py +2 -2
- eodag/plugins/crunch/filter_latest_tpl_name.py +1 -1
- eodag/plugins/download/aws.py +46 -42
- eodag/plugins/download/base.py +13 -14
- eodag/plugins/download/http.py +65 -65
- eodag/plugins/manager.py +28 -29
- eodag/plugins/search/__init__.py +6 -4
- eodag/plugins/search/base.py +131 -80
- eodag/plugins/search/build_search_result.py +245 -173
- eodag/plugins/search/cop_marine.py +87 -56
- eodag/plugins/search/csw.py +47 -37
- eodag/plugins/search/qssearch.py +653 -429
- eodag/plugins/search/stac_list_assets.py +1 -1
- eodag/plugins/search/static_stac_search.py +43 -44
- eodag/resources/{product_types.yml → collections.yml} +2594 -2453
- eodag/resources/ext_collections.json +1 -1
- eodag/resources/ext_product_types.json +1 -1
- eodag/resources/providers.yml +2706 -2733
- eodag/resources/stac_provider.yml +50 -92
- eodag/resources/user_conf_template.yml +9 -0
- eodag/types/__init__.py +2 -0
- eodag/types/queryables.py +70 -91
- eodag/types/search_args.py +1 -1
- eodag/utils/__init__.py +97 -21
- eodag/utils/dates.py +0 -12
- eodag/utils/exceptions.py +6 -6
- eodag/utils/free_text_search.py +3 -3
- eodag/utils/repr.py +2 -0
- {eodag-3.10.1.dist-info → eodag-4.0.0a2.dist-info}/METADATA +13 -99
- eodag-4.0.0a2.dist-info/RECORD +93 -0
- {eodag-3.10.1.dist-info → eodag-4.0.0a2.dist-info}/entry_points.txt +0 -4
- eodag/plugins/authentication/oauth.py +0 -60
- eodag/plugins/download/creodias_s3.py +0 -71
- eodag/plugins/download/s3rest.py +0 -351
- eodag/plugins/search/data_request_search.py +0 -565
- eodag/resources/stac.yml +0 -294
- eodag/resources/stac_api.yml +0 -2105
- eodag/rest/__init__.py +0 -24
- eodag/rest/cache.py +0 -70
- eodag/rest/config.py +0 -67
- eodag/rest/constants.py +0 -26
- eodag/rest/core.py +0 -764
- eodag/rest/errors.py +0 -210
- eodag/rest/server.py +0 -604
- eodag/rest/server.wsgi +0 -6
- eodag/rest/stac.py +0 -1032
- eodag/rest/templates/README +0 -1
- eodag/rest/types/__init__.py +0 -18
- eodag/rest/types/collections_search.py +0 -44
- eodag/rest/types/eodag_search.py +0 -386
- eodag/rest/types/queryables.py +0 -174
- eodag/rest/types/stac_search.py +0 -272
- eodag/rest/utils/__init__.py +0 -207
- eodag/rest/utils/cql_evaluate.py +0 -119
- eodag/rest/utils/rfc3339.py +0 -64
- eodag-3.10.1.dist-info/RECORD +0 -116
- {eodag-3.10.1.dist-info → eodag-4.0.0a2.dist-info}/WHEEL +0 -0
- {eodag-3.10.1.dist-info → eodag-4.0.0a2.dist-info}/licenses/LICENSE +0 -0
- {eodag-3.10.1.dist-info → eodag-4.0.0a2.dist-info}/top_level.txt +0 -0
eodag/api/core.py
CHANGED
|
@@ -23,14 +23,17 @@ import os
|
|
|
23
23
|
import re
|
|
24
24
|
import shutil
|
|
25
25
|
import tempfile
|
|
26
|
+
import warnings
|
|
27
|
+
from collections import deque
|
|
26
28
|
from importlib.metadata import version
|
|
27
29
|
from importlib.resources import files as res_files
|
|
28
|
-
from operator import itemgetter
|
|
30
|
+
from operator import attrgetter, itemgetter
|
|
29
31
|
from typing import TYPE_CHECKING, Any, Iterator, Optional, Union
|
|
30
32
|
|
|
31
33
|
import geojson
|
|
32
|
-
import yaml
|
|
34
|
+
import yaml
|
|
33
35
|
|
|
36
|
+
from eodag.api.collection import Collection, CollectionsDict, CollectionsList
|
|
34
37
|
from eodag.api.product.metadata_mapping import (
|
|
35
38
|
NOT_AVAILABLE,
|
|
36
39
|
mtd_cfg_as_conversion_and_querypath,
|
|
@@ -41,7 +44,7 @@ from eodag.config import (
|
|
|
41
44
|
PluginConfig,
|
|
42
45
|
SimpleYamlProxyConfig,
|
|
43
46
|
credentials_in_auth,
|
|
44
|
-
|
|
47
|
+
get_ext_collections_conf,
|
|
45
48
|
load_default_config,
|
|
46
49
|
load_stac_provider_config,
|
|
47
50
|
load_yml_config,
|
|
@@ -53,7 +56,10 @@ from eodag.config import (
|
|
|
53
56
|
)
|
|
54
57
|
from eodag.plugins.manager import PluginManager
|
|
55
58
|
from eodag.plugins.search import PreparedSearch
|
|
56
|
-
from eodag.plugins.search.build_search_result import
|
|
59
|
+
from eodag.plugins.search.build_search_result import (
|
|
60
|
+
ALLOWED_KEYWORDS as ECMWF_ALLOWED_KEYWORDS,
|
|
61
|
+
)
|
|
62
|
+
from eodag.plugins.search.build_search_result import ECMWF_PREFIX, MeteoblueSearch
|
|
57
63
|
from eodag.plugins.search.qssearch import PostJsonSearch
|
|
58
64
|
from eodag.types import model_fields_to_annotated
|
|
59
65
|
from eodag.types.queryables import CommonQueryables, Queryables, QueryablesDict
|
|
@@ -63,10 +69,8 @@ from eodag.utils import (
|
|
|
63
69
|
DEFAULT_ITEMS_PER_PAGE,
|
|
64
70
|
DEFAULT_MAX_ITEMS_PER_PAGE,
|
|
65
71
|
DEFAULT_PAGE,
|
|
66
|
-
|
|
72
|
+
GENERIC_COLLECTION,
|
|
67
73
|
GENERIC_STAC_PROVIDER,
|
|
68
|
-
HTTP_REQ_TIMEOUT,
|
|
69
|
-
MockResponse,
|
|
70
74
|
_deprecated,
|
|
71
75
|
get_geometry_from_various,
|
|
72
76
|
makedirs,
|
|
@@ -78,11 +82,12 @@ from eodag.utils.dates import rfc3339_str_to_datetime
|
|
|
78
82
|
from eodag.utils.env import is_env_var_true
|
|
79
83
|
from eodag.utils.exceptions import (
|
|
80
84
|
AuthenticationError,
|
|
81
|
-
|
|
85
|
+
NoMatchingCollection,
|
|
82
86
|
PluginImplementationError,
|
|
83
87
|
RequestError,
|
|
84
|
-
|
|
88
|
+
UnsupportedCollection,
|
|
85
89
|
UnsupportedProvider,
|
|
90
|
+
ValidationError,
|
|
86
91
|
)
|
|
87
92
|
from eodag.utils.free_text_search import compile_free_text_query
|
|
88
93
|
from eodag.utils.stac_reader import fetch_stac_items
|
|
@@ -114,10 +119,11 @@ class EODataAccessGateway:
|
|
|
114
119
|
user_conf_file_path: Optional[str] = None,
|
|
115
120
|
locations_conf_path: Optional[str] = None,
|
|
116
121
|
) -> None:
|
|
117
|
-
|
|
118
|
-
res_files("eodag") / "resources" / "
|
|
122
|
+
collections_config_path = os.getenv("EODAG_COLLECTIONS_CFG_FILE") or str(
|
|
123
|
+
res_files("eodag") / "resources" / "collections.yml"
|
|
119
124
|
)
|
|
120
|
-
|
|
125
|
+
collections_config_dict = SimpleYamlProxyConfig(collections_config_path).source
|
|
126
|
+
self.collections_config = self._collections_config_init(collections_config_dict)
|
|
121
127
|
self.providers_config = load_default_config()
|
|
122
128
|
|
|
123
129
|
env_var_cfg_dir = "EODAG_CFG_DIR"
|
|
@@ -169,8 +175,8 @@ class EODataAccessGateway:
|
|
|
169
175
|
share_credentials(self.providers_config)
|
|
170
176
|
|
|
171
177
|
# init updated providers conf
|
|
172
|
-
strict_mode = is_env_var_true("
|
|
173
|
-
|
|
178
|
+
strict_mode = is_env_var_true("EODAG_STRICT_COLLECTIONS")
|
|
179
|
+
available_collections = set(self.collections_config.keys())
|
|
174
180
|
|
|
175
181
|
for provider in self.providers_config.keys():
|
|
176
182
|
provider_config_init(
|
|
@@ -178,11 +184,9 @@ class EODataAccessGateway:
|
|
|
178
184
|
load_stac_provider_config(),
|
|
179
185
|
)
|
|
180
186
|
|
|
181
|
-
self.
|
|
182
|
-
provider,
|
|
187
|
+
self._sync_provider_collections(
|
|
188
|
+
provider, available_collections, strict_mode
|
|
183
189
|
)
|
|
184
|
-
# init product types configuration
|
|
185
|
-
self._product_types_config_init()
|
|
186
190
|
|
|
187
191
|
# re-build _plugins_manager using up-to-date providers_config
|
|
188
192
|
self._plugins_manager.rebuild(self.providers_config)
|
|
@@ -225,26 +229,35 @@ class EODataAccessGateway:
|
|
|
225
229
|
)
|
|
226
230
|
self.set_locations_conf(locations_conf_path)
|
|
227
231
|
|
|
228
|
-
def
|
|
229
|
-
|
|
230
|
-
|
|
231
|
-
|
|
232
|
+
def _collections_config_init(
|
|
233
|
+
self, collections_config_dict: dict[str, Any]
|
|
234
|
+
) -> CollectionsDict:
|
|
235
|
+
"""Initialize collections configuration.
|
|
232
236
|
|
|
233
|
-
|
|
237
|
+
:param collections_config_dict: The collections config as a dictionary
|
|
238
|
+
"""
|
|
239
|
+
# Turn the collections config from a dict into a CollectionsDict() object
|
|
240
|
+
collections = [
|
|
241
|
+
Collection.create_with_dag(self, id=col, **col_f)
|
|
242
|
+
for col, col_f in collections_config_dict.items()
|
|
243
|
+
]
|
|
244
|
+
return CollectionsDict(collections)
|
|
245
|
+
|
|
246
|
+
def _sync_provider_collections(
|
|
234
247
|
self,
|
|
235
248
|
provider: str,
|
|
236
|
-
|
|
249
|
+
available_collections: set[str],
|
|
237
250
|
strict_mode: bool,
|
|
238
251
|
) -> None:
|
|
239
252
|
"""
|
|
240
|
-
Synchronize
|
|
253
|
+
Synchronize collections for a provider based on strict or permissive mode.
|
|
241
254
|
|
|
242
|
-
In strict mode, removes
|
|
243
|
-
In permissive mode, adds empty
|
|
255
|
+
In strict mode, removes collections not in available_collections.
|
|
256
|
+
In permissive mode, adds empty collection configs for missing types.
|
|
244
257
|
|
|
245
|
-
:param provider: The provider name whose
|
|
246
|
-
:param
|
|
247
|
-
:param strict_mode: If True, remove unknown
|
|
258
|
+
:param provider: The provider name whose collections should be synchronized.
|
|
259
|
+
:param available_collections: The set of available collection IDs.
|
|
260
|
+
:param strict_mode: If True, remove unknown collections; if False, add empty configs for them.
|
|
248
261
|
:returns: None
|
|
249
262
|
"""
|
|
250
263
|
provider_products = self.providers_config[provider].products
|
|
@@ -252,33 +265,30 @@ class EODataAccessGateway:
|
|
|
252
265
|
products_to_add: list[str] = []
|
|
253
266
|
|
|
254
267
|
for product_id in provider_products:
|
|
255
|
-
if product_id ==
|
|
268
|
+
if product_id == GENERIC_COLLECTION:
|
|
256
269
|
continue
|
|
257
270
|
|
|
258
|
-
if product_id not in
|
|
271
|
+
if product_id not in available_collections:
|
|
259
272
|
if strict_mode:
|
|
260
273
|
products_to_remove.append(product_id)
|
|
261
274
|
continue
|
|
262
275
|
|
|
263
|
-
empty_product =
|
|
264
|
-
|
|
265
|
-
|
|
266
|
-
|
|
267
|
-
self.product_types_config.source[
|
|
268
|
-
product_id
|
|
269
|
-
] = empty_product # will update available_product_types
|
|
276
|
+
empty_product = Collection.create_with_dag(
|
|
277
|
+
self, id=product_id, title=product_id, description=NOT_AVAILABLE
|
|
278
|
+
)
|
|
279
|
+
self.collections_config[product_id] = empty_product
|
|
270
280
|
products_to_add.append(product_id)
|
|
271
281
|
|
|
272
282
|
if products_to_add:
|
|
273
283
|
logger.debug(
|
|
274
|
-
"
|
|
284
|
+
"Collections permissive mode, %s added (provider %s)",
|
|
275
285
|
", ".join(products_to_add),
|
|
276
286
|
provider,
|
|
277
287
|
)
|
|
278
288
|
|
|
279
289
|
if products_to_remove:
|
|
280
290
|
logger.debug(
|
|
281
|
-
"
|
|
291
|
+
"Collections strict mode, ignoring %s (provider %s)",
|
|
282
292
|
", ".join(products_to_remove),
|
|
283
293
|
provider,
|
|
284
294
|
)
|
|
@@ -357,9 +367,9 @@ class EODataAccessGateway:
|
|
|
357
367
|
self.providers_config[provider],
|
|
358
368
|
load_stac_provider_config(),
|
|
359
369
|
)
|
|
360
|
-
setattr(self.providers_config[provider], "
|
|
370
|
+
setattr(self.providers_config[provider], "collections_fetched", False)
|
|
361
371
|
# re-create _plugins_manager using up-to-date providers_config
|
|
362
|
-
self._plugins_manager.
|
|
372
|
+
self._plugins_manager.build_collection_to_provider_config_map()
|
|
363
373
|
|
|
364
374
|
def add_provider(
|
|
365
375
|
self,
|
|
@@ -368,7 +378,7 @@ class EODataAccessGateway:
|
|
|
368
378
|
priority: Optional[int] = None,
|
|
369
379
|
search: dict[str, Any] = {"type": "StacSearch"},
|
|
370
380
|
products: dict[str, Any] = {
|
|
371
|
-
|
|
381
|
+
GENERIC_COLLECTION: {"_collection": "{collection}"}
|
|
372
382
|
},
|
|
373
383
|
download: dict[str, Any] = {"type": "HTTPDownload", "auth_error_code": 401},
|
|
374
384
|
**kwargs: dict[str, Any],
|
|
@@ -379,14 +389,14 @@ class EODataAccessGateway:
|
|
|
379
389
|
updated (not replaced), with user provided ones:
|
|
380
390
|
|
|
381
391
|
* ``search`` : ``{"type": "StacSearch"}``
|
|
382
|
-
* ``products`` : ``{"
|
|
392
|
+
* ``products`` : ``{"GENERIC_COLLECTION": {"_collection": "{collection}"}}``
|
|
383
393
|
* ``download`` : ``{"type": "HTTPDownload", "auth_error_code": 401}``
|
|
384
394
|
|
|
385
395
|
:param name: Name of provider
|
|
386
396
|
:param url: Provider url, also used as ``search["api_endpoint"]`` if not defined
|
|
387
397
|
:param priority: Provider priority. If None, provider will be set as preferred (highest priority)
|
|
388
398
|
:param search: Search :class:`~eodag.config.PluginConfig` mapping
|
|
389
|
-
:param products: Provider
|
|
399
|
+
:param products: Provider collections mapping
|
|
390
400
|
:param download: Download :class:`~eodag.config.PluginConfig` mapping
|
|
391
401
|
:param kwargs: Additional :class:`~eodag.config.ProviderConfig` mapping
|
|
392
402
|
"""
|
|
@@ -395,7 +405,7 @@ class EODataAccessGateway:
|
|
|
395
405
|
"url": url,
|
|
396
406
|
"search": {"type": "StacSearch", **search},
|
|
397
407
|
"products": {
|
|
398
|
-
|
|
408
|
+
GENERIC_COLLECTION: {"_collection": "{collection}"},
|
|
399
409
|
**products,
|
|
400
410
|
},
|
|
401
411
|
"download": {
|
|
@@ -541,23 +551,23 @@ class EODataAccessGateway:
|
|
|
541
551
|
)
|
|
542
552
|
self.locations_config = []
|
|
543
553
|
|
|
544
|
-
def
|
|
554
|
+
def list_collections(
|
|
545
555
|
self, provider: Optional[str] = None, fetch_providers: bool = True
|
|
546
|
-
) ->
|
|
547
|
-
"""Lists supported
|
|
556
|
+
) -> CollectionsList:
|
|
557
|
+
"""Lists supported collections.
|
|
548
558
|
|
|
549
559
|
:param provider: (optional) The name of a provider that must support the product
|
|
550
560
|
types we are about to list
|
|
551
561
|
:param fetch_providers: (optional) Whether to fetch providers for new product
|
|
552
562
|
types or not
|
|
553
|
-
:returns: The list of the
|
|
563
|
+
:returns: The list of the collections that can be accessed using eodag.
|
|
554
564
|
:raises: :class:`~eodag.utils.exceptions.UnsupportedProvider`
|
|
555
565
|
"""
|
|
556
566
|
if fetch_providers:
|
|
557
|
-
# First, update
|
|
558
|
-
self.
|
|
567
|
+
# First, update collections list if possible
|
|
568
|
+
self.fetch_collections_list(provider=provider)
|
|
559
569
|
|
|
560
|
-
|
|
570
|
+
collections: CollectionsList = CollectionsList([])
|
|
561
571
|
|
|
562
572
|
providers_configs = (
|
|
563
573
|
list(self.providers_config.values())
|
|
@@ -575,31 +585,29 @@ class EODataAccessGateway:
|
|
|
575
585
|
)
|
|
576
586
|
|
|
577
587
|
for p in providers_configs:
|
|
578
|
-
for
|
|
579
|
-
if
|
|
588
|
+
for collection_id in p.products:
|
|
589
|
+
if collection_id == GENERIC_COLLECTION:
|
|
580
590
|
continue
|
|
581
591
|
|
|
582
|
-
|
|
583
|
-
|
|
584
|
-
|
|
585
|
-
|
|
586
|
-
|
|
587
|
-
if product_type not in product_types:
|
|
588
|
-
product_types.append(product_type)
|
|
592
|
+
if (
|
|
593
|
+
collection := self.collections_config[collection_id]
|
|
594
|
+
) not in collections:
|
|
595
|
+
collections.append(collection)
|
|
589
596
|
|
|
590
|
-
# Return the
|
|
591
|
-
|
|
597
|
+
# Return the collections sorted in lexicographic order of their id
|
|
598
|
+
collections.sort(key=attrgetter("id"))
|
|
599
|
+
return collections
|
|
592
600
|
|
|
593
|
-
def
|
|
594
|
-
"""Fetch
|
|
601
|
+
def fetch_collections_list(self, provider: Optional[str] = None) -> None:
|
|
602
|
+
"""Fetch collections list and update if needed.
|
|
595
603
|
|
|
596
|
-
If strict mode is enabled (by setting the ``
|
|
597
|
-
to a truthy value), this method will not fetch or update
|
|
604
|
+
If strict mode is enabled (by setting the ``EODAG_STRICT_COLLECTIONS`` environment variable
|
|
605
|
+
to a truthy value), this method will not fetch or update collections and will return immediately.
|
|
598
606
|
|
|
599
|
-
:param provider: The name of a provider or provider-group for which
|
|
607
|
+
:param provider: The name of a provider or provider-group for which collections
|
|
600
608
|
list should be updated. Defaults to all providers (None value).
|
|
601
609
|
"""
|
|
602
|
-
strict_mode = is_env_var_true("
|
|
610
|
+
strict_mode = is_env_var_true("EODAG_STRICT_COLLECTIONS")
|
|
603
611
|
if strict_mode:
|
|
604
612
|
return
|
|
605
613
|
|
|
@@ -613,7 +621,7 @@ class EODataAccessGateway:
|
|
|
613
621
|
]
|
|
614
622
|
if providers_to_fetch:
|
|
615
623
|
logger.info(
|
|
616
|
-
f"Fetch
|
|
624
|
+
f"Fetch collections for {provider} group: {', '.join(providers_to_fetch)}"
|
|
617
625
|
)
|
|
618
626
|
else:
|
|
619
627
|
return None
|
|
@@ -622,7 +630,7 @@ class EODataAccessGateway:
|
|
|
622
630
|
|
|
623
631
|
# providers discovery confs that are fetchable
|
|
624
632
|
providers_discovery_configs_fetchable: dict[str, Any] = {}
|
|
625
|
-
# check if any provider has not already been fetched for
|
|
633
|
+
# check if any provider has not already been fetched for collections
|
|
626
634
|
already_fetched = True
|
|
627
635
|
for provider_to_fetch in providers_to_fetch:
|
|
628
636
|
provider_config = self.providers_config[provider_to_fetch]
|
|
@@ -633,45 +641,43 @@ class EODataAccessGateway:
|
|
|
633
641
|
provider_search_config = provider_config.api
|
|
634
642
|
else:
|
|
635
643
|
continue
|
|
636
|
-
discovery_conf = getattr(
|
|
637
|
-
provider_search_config, "discover_product_types", {}
|
|
638
|
-
)
|
|
644
|
+
discovery_conf = getattr(provider_search_config, "discover_collections", {})
|
|
639
645
|
if discovery_conf.get("fetch_url"):
|
|
640
646
|
providers_discovery_configs_fetchable[
|
|
641
647
|
provider_to_fetch
|
|
642
648
|
] = discovery_conf
|
|
643
|
-
if not getattr(provider_config, "
|
|
649
|
+
if not getattr(provider_config, "collections_fetched", False):
|
|
644
650
|
already_fetched = False
|
|
645
651
|
|
|
646
652
|
if not already_fetched:
|
|
647
|
-
# get
|
|
648
|
-
|
|
649
|
-
if
|
|
650
|
-
|
|
651
|
-
|
|
653
|
+
# get ext_collections conf
|
|
654
|
+
ext_collections_cfg_file = os.getenv("EODAG_EXT_COLLECTIONS_CFG_FILE")
|
|
655
|
+
if ext_collections_cfg_file is not None:
|
|
656
|
+
ext_collections_conf = get_ext_collections_conf(
|
|
657
|
+
ext_collections_cfg_file
|
|
652
658
|
)
|
|
653
659
|
else:
|
|
654
|
-
|
|
660
|
+
ext_collections_conf = get_ext_collections_conf()
|
|
655
661
|
|
|
656
|
-
if not
|
|
657
|
-
# empty
|
|
658
|
-
|
|
659
|
-
self.
|
|
662
|
+
if not ext_collections_conf:
|
|
663
|
+
# empty ext_collections conf
|
|
664
|
+
ext_collections_conf = (
|
|
665
|
+
self.discover_collections(provider=provider) or {}
|
|
660
666
|
)
|
|
661
667
|
|
|
662
|
-
# update eodag
|
|
663
|
-
self.
|
|
668
|
+
# update eodag collections list with new conf
|
|
669
|
+
self.update_collections_list(ext_collections_conf)
|
|
664
670
|
|
|
665
671
|
# Compare current provider with default one to see if it has been modified
|
|
666
|
-
# and
|
|
672
|
+
# and collections list would need to be fetched
|
|
667
673
|
|
|
668
|
-
# get
|
|
674
|
+
# get ext_collections conf for user modified providers
|
|
669
675
|
default_providers_config = load_default_config()
|
|
670
676
|
for (
|
|
671
677
|
provider,
|
|
672
678
|
user_discovery_conf,
|
|
673
679
|
) in providers_discovery_configs_fetchable.items():
|
|
674
|
-
# default
|
|
680
|
+
# default discover_collections conf
|
|
675
681
|
if provider in default_providers_config:
|
|
676
682
|
default_provider_config = default_providers_config[provider]
|
|
677
683
|
if hasattr(default_provider_config, "search"):
|
|
@@ -681,7 +687,7 @@ class EODataAccessGateway:
|
|
|
681
687
|
else:
|
|
682
688
|
continue
|
|
683
689
|
default_discovery_conf = getattr(
|
|
684
|
-
default_provider_search_config, "
|
|
690
|
+
default_provider_search_config, "discover_collections", {}
|
|
685
691
|
)
|
|
686
692
|
# compare confs
|
|
687
693
|
if default_discovery_conf["result_type"] == "json" and isinstance(
|
|
@@ -696,22 +702,22 @@ class EODataAccessGateway:
|
|
|
696
702
|
},
|
|
697
703
|
**mtd_cfg_as_conversion_and_querypath(
|
|
698
704
|
dict(
|
|
699
|
-
|
|
700
|
-
"
|
|
705
|
+
generic_collection_id=default_discovery_conf[
|
|
706
|
+
"generic_collection_id"
|
|
701
707
|
]
|
|
702
708
|
)
|
|
703
709
|
),
|
|
704
710
|
**dict(
|
|
705
|
-
|
|
711
|
+
generic_collection_parsable_properties=mtd_cfg_as_conversion_and_querypath(
|
|
706
712
|
default_discovery_conf[
|
|
707
|
-
"
|
|
713
|
+
"generic_collection_parsable_properties"
|
|
708
714
|
]
|
|
709
715
|
)
|
|
710
716
|
),
|
|
711
717
|
**dict(
|
|
712
|
-
|
|
718
|
+
generic_collection_parsable_metadata=mtd_cfg_as_conversion_and_querypath(
|
|
713
719
|
default_discovery_conf[
|
|
714
|
-
"
|
|
720
|
+
"generic_collection_parsable_metadata"
|
|
715
721
|
]
|
|
716
722
|
)
|
|
717
723
|
),
|
|
@@ -723,33 +729,33 @@ class EODataAccessGateway:
|
|
|
723
729
|
or user_discovery_conf == default_discovery_conf_parsed
|
|
724
730
|
) and (
|
|
725
731
|
not default_discovery_conf.get("fetch_url")
|
|
726
|
-
or "
|
|
727
|
-
or "
|
|
732
|
+
or "ext_collections_conf" not in locals()
|
|
733
|
+
or "ext_collections_conf" in locals()
|
|
728
734
|
and (
|
|
729
|
-
provider in
|
|
730
|
-
or len(
|
|
735
|
+
provider in ext_collections_conf
|
|
736
|
+
or len(ext_collections_conf.keys()) == 0
|
|
731
737
|
)
|
|
732
738
|
):
|
|
733
739
|
continue
|
|
734
740
|
# providers not skipped here should be user-modified
|
|
735
|
-
# or not in
|
|
741
|
+
# or not in ext_collections_conf (if eodag system conf != eodag conf used for ext_collections_conf)
|
|
736
742
|
|
|
737
743
|
if not already_fetched:
|
|
738
|
-
# discover
|
|
739
|
-
|
|
740
|
-
self.
|
|
744
|
+
# discover collections for user configured provider
|
|
745
|
+
provider_ext_collections_conf = (
|
|
746
|
+
self.discover_collections(provider=provider) or {}
|
|
741
747
|
)
|
|
742
|
-
# update eodag
|
|
743
|
-
self.
|
|
748
|
+
# update eodag collections list with new conf
|
|
749
|
+
self.update_collections_list(provider_ext_collections_conf)
|
|
744
750
|
|
|
745
|
-
def
|
|
751
|
+
def discover_collections(
|
|
746
752
|
self, provider: Optional[str] = None
|
|
747
753
|
) -> Optional[dict[str, Any]]:
|
|
748
|
-
"""Fetch providers for
|
|
754
|
+
"""Fetch providers for collections
|
|
749
755
|
|
|
750
756
|
:param provider: The name of a provider or provider-group to fetch. Defaults to
|
|
751
757
|
all providers (None value).
|
|
752
|
-
:returns: external
|
|
758
|
+
:returns: external collections configuration
|
|
753
759
|
"""
|
|
754
760
|
grouped_providers = [
|
|
755
761
|
p
|
|
@@ -758,13 +764,13 @@ class EODataAccessGateway:
|
|
|
758
764
|
]
|
|
759
765
|
if provider and provider not in self.providers_config and grouped_providers:
|
|
760
766
|
logger.info(
|
|
761
|
-
f"Discover
|
|
767
|
+
f"Discover collections for {provider} group: {', '.join(grouped_providers)}"
|
|
762
768
|
)
|
|
763
769
|
elif provider and provider not in self.providers_config:
|
|
764
770
|
raise UnsupportedProvider(
|
|
765
771
|
f"The requested provider is not (yet) supported: {provider}"
|
|
766
772
|
)
|
|
767
|
-
|
|
773
|
+
ext_collections_conf: dict[str, Any] = {}
|
|
768
774
|
providers_to_fetch = [
|
|
769
775
|
p
|
|
770
776
|
for p in (
|
|
@@ -785,14 +791,14 @@ class EODataAccessGateway:
|
|
|
785
791
|
search_plugin_config = self.providers_config[provider].api
|
|
786
792
|
else:
|
|
787
793
|
return None
|
|
788
|
-
if getattr(search_plugin_config, "
|
|
794
|
+
if getattr(search_plugin_config, "discover_collections", {}).get(
|
|
789
795
|
"fetch_url", None
|
|
790
796
|
):
|
|
791
797
|
search_plugin: Union[Search, Api] = next(
|
|
792
798
|
self._plugins_manager.get_search_plugins(provider=provider)
|
|
793
799
|
)
|
|
794
800
|
# check after plugin init if still fetchable
|
|
795
|
-
if not getattr(search_plugin.config, "
|
|
801
|
+
if not getattr(search_plugin.config, "discover_collections", {}).get(
|
|
796
802
|
"fetch_url"
|
|
797
803
|
):
|
|
798
804
|
continue
|
|
@@ -806,26 +812,26 @@ class EODataAccessGateway:
|
|
|
806
812
|
kwargs["auth"] = auth
|
|
807
813
|
else:
|
|
808
814
|
logger.debug(
|
|
809
|
-
f"Could not authenticate on {provider} for
|
|
815
|
+
f"Could not authenticate on {provider} for collections discovery"
|
|
810
816
|
)
|
|
811
|
-
|
|
817
|
+
ext_collections_conf[provider] = None
|
|
812
818
|
continue
|
|
813
819
|
|
|
814
|
-
|
|
820
|
+
ext_collections_conf[provider] = search_plugin.discover_collections(
|
|
815
821
|
**kwargs
|
|
816
822
|
)
|
|
817
823
|
|
|
818
|
-
return sort_dict(
|
|
824
|
+
return sort_dict(ext_collections_conf)
|
|
819
825
|
|
|
820
|
-
def
|
|
821
|
-
self,
|
|
826
|
+
def update_collections_list(
|
|
827
|
+
self, ext_collections_conf: dict[str, Optional[dict[str, dict[str, Any]]]]
|
|
822
828
|
) -> None:
|
|
823
|
-
"""Update eodag
|
|
829
|
+
"""Update eodag collections list
|
|
824
830
|
|
|
825
|
-
:param
|
|
831
|
+
:param ext_collections_conf: external collections configuration
|
|
826
832
|
"""
|
|
827
|
-
for provider,
|
|
828
|
-
if
|
|
833
|
+
for provider, new_collections_conf in ext_collections_conf.items():
|
|
834
|
+
if new_collections_conf and provider in self.providers_config:
|
|
829
835
|
try:
|
|
830
836
|
search_plugin_config = getattr(
|
|
831
837
|
self.providers_config[provider], "search", None
|
|
@@ -833,77 +839,108 @@ class EODataAccessGateway:
|
|
|
833
839
|
if search_plugin_config is None:
|
|
834
840
|
continue
|
|
835
841
|
if not getattr(
|
|
836
|
-
search_plugin_config, "
|
|
842
|
+
search_plugin_config, "discover_collections", {}
|
|
837
843
|
).get("fetch_url"):
|
|
838
|
-
# conf has been updated and provider
|
|
844
|
+
# conf has been updated and provider collections are no more discoverable
|
|
839
845
|
continue
|
|
840
846
|
provider_products_config = (
|
|
841
847
|
self.providers_config[provider].products or {}
|
|
842
848
|
)
|
|
843
849
|
except UnsupportedProvider:
|
|
844
850
|
logger.debug(
|
|
845
|
-
"Ignoring external
|
|
851
|
+
"Ignoring external collections for unknown provider %s",
|
|
846
852
|
provider,
|
|
847
853
|
)
|
|
848
854
|
continue
|
|
849
|
-
|
|
855
|
+
new_collections: list[str] = []
|
|
856
|
+
bad_formatted_col_count = 0
|
|
850
857
|
for (
|
|
851
|
-
|
|
852
|
-
|
|
853
|
-
) in
|
|
854
|
-
if
|
|
855
|
-
for
|
|
858
|
+
new_collection,
|
|
859
|
+
new_collection_conf,
|
|
860
|
+
) in new_collections_conf["providers_config"].items():
|
|
861
|
+
if new_collection not in provider_products_config:
|
|
862
|
+
for existing_collection in provider_products_config.copy():
|
|
856
863
|
# compare parsed extracted conf (without metadata_mapping entry)
|
|
857
864
|
unparsable_keys = (
|
|
858
|
-
search_plugin_config.
|
|
859
|
-
"
|
|
865
|
+
search_plugin_config.discover_collections.get(
|
|
866
|
+
"generic_collection_unparsable_properties", {}
|
|
860
867
|
).keys()
|
|
861
868
|
)
|
|
862
|
-
|
|
869
|
+
new_parsed_collections_conf = {
|
|
863
870
|
k: v
|
|
864
|
-
for k, v in
|
|
871
|
+
for k, v in new_collection_conf.items()
|
|
865
872
|
if k not in unparsable_keys
|
|
866
873
|
}
|
|
867
874
|
if (
|
|
868
|
-
|
|
869
|
-
<= provider_products_config[
|
|
870
|
-
existing_product_type
|
|
871
|
-
].items()
|
|
875
|
+
new_parsed_collections_conf.items()
|
|
876
|
+
<= provider_products_config[existing_collection].items()
|
|
872
877
|
):
|
|
873
|
-
#
|
|
878
|
+
# new_collections_conf is a subset on an existing conf
|
|
874
879
|
break
|
|
875
880
|
else:
|
|
876
|
-
|
|
877
|
-
|
|
878
|
-
|
|
879
|
-
|
|
880
|
-
|
|
881
|
-
|
|
882
|
-
|
|
883
|
-
|
|
884
|
-
|
|
885
|
-
|
|
886
|
-
|
|
887
|
-
|
|
881
|
+
try:
|
|
882
|
+
# new_collection_conf does not already exist, append it
|
|
883
|
+
# to self.collections_config
|
|
884
|
+
self.collections_config[
|
|
885
|
+
new_collection
|
|
886
|
+
] = Collection.create_with_dag(
|
|
887
|
+
self,
|
|
888
|
+
id=new_collection,
|
|
889
|
+
**new_collections_conf["collections_config"][
|
|
890
|
+
new_collection
|
|
891
|
+
],
|
|
892
|
+
)
|
|
893
|
+
except ValidationError:
|
|
894
|
+
# skip collection if there is a problem with its id (missing or not a string)
|
|
895
|
+
logger.debug(
|
|
896
|
+
(
|
|
897
|
+
"Collection %s has been pruned on provider %s "
|
|
898
|
+
"because its id was incorrectly parsed for eodag"
|
|
899
|
+
),
|
|
900
|
+
new_collection,
|
|
901
|
+
provider,
|
|
902
|
+
)
|
|
903
|
+
else:
|
|
904
|
+
# to provider_products_config
|
|
905
|
+
provider_products_config[
|
|
906
|
+
new_collection
|
|
907
|
+
] = new_collection_conf
|
|
908
|
+
ext_collections_conf[provider] = new_collections_conf
|
|
909
|
+
new_collections.append(new_collection)
|
|
910
|
+
# increase the increment if the new collection had
|
|
911
|
+
# bad formatted attributes in the external config
|
|
912
|
+
dumped_collection = self.collections_config[
|
|
913
|
+
new_collection
|
|
914
|
+
].model_dump()
|
|
915
|
+
dumped_ext_conf_col = {
|
|
916
|
+
**dumped_collection,
|
|
917
|
+
**new_collections_conf["collections_config"][
|
|
918
|
+
new_collection
|
|
919
|
+
],
|
|
888
920
|
}
|
|
889
|
-
|
|
890
|
-
|
|
891
|
-
|
|
892
|
-
if new_product_types:
|
|
921
|
+
if dumped_ext_conf_col != dumped_collection:
|
|
922
|
+
bad_formatted_col_count += 1
|
|
923
|
+
if new_collections:
|
|
893
924
|
logger.debug(
|
|
894
|
-
|
|
925
|
+
"Added %s collections for %s", len(new_collections), provider
|
|
895
926
|
)
|
|
927
|
+
if bad_formatted_col_count > 0:
|
|
928
|
+
logger.debug(
|
|
929
|
+
"bad formatted attributes skipped for %s collection(s) on %s",
|
|
930
|
+
bad_formatted_col_count,
|
|
931
|
+
provider,
|
|
932
|
+
)
|
|
896
933
|
|
|
897
934
|
elif provider not in self.providers_config:
|
|
898
935
|
# unknown provider
|
|
899
936
|
continue
|
|
900
|
-
self.providers_config[provider].
|
|
937
|
+
self.providers_config[provider].collections_fetched = True
|
|
901
938
|
|
|
902
939
|
# re-create _plugins_manager using up-to-date providers_config
|
|
903
|
-
self._plugins_manager.
|
|
940
|
+
self._plugins_manager.build_collection_to_provider_config_map()
|
|
904
941
|
|
|
905
942
|
def available_providers(
|
|
906
|
-
self,
|
|
943
|
+
self, collection: Optional[str] = None, by_group: bool = False
|
|
907
944
|
) -> list[str]:
|
|
908
945
|
"""Gives the sorted list of the available providers or groups
|
|
909
946
|
|
|
@@ -911,17 +948,17 @@ class EODataAccessGateway:
|
|
|
911
948
|
and then alphabetically in ascending order for providers or groups with the same
|
|
912
949
|
priority level.
|
|
913
950
|
|
|
914
|
-
:param
|
|
951
|
+
:param collection: (optional) Only list providers configured for this collection
|
|
915
952
|
:param by_group: (optional) If set to True, list groups when available instead
|
|
916
953
|
of providers, mixed with other providers
|
|
917
954
|
:returns: the sorted list of the available providers or groups
|
|
918
955
|
"""
|
|
919
956
|
|
|
920
|
-
if
|
|
957
|
+
if collection:
|
|
921
958
|
providers = [
|
|
922
959
|
(v.group if by_group and hasattr(v, "group") else k, v.priority)
|
|
923
960
|
for k, v in self.providers_config.items()
|
|
924
|
-
if
|
|
961
|
+
if collection in getattr(v, "products", {}).keys()
|
|
925
962
|
]
|
|
926
963
|
else:
|
|
927
964
|
providers = [
|
|
@@ -943,97 +980,103 @@ class EODataAccessGateway:
|
|
|
943
980
|
# Return only the names of the providers or groups
|
|
944
981
|
return [name for name, _ in providers]
|
|
945
982
|
|
|
946
|
-
def
|
|
947
|
-
"""Return the
|
|
983
|
+
def get_collection_from_alias(self, alias_or_id: str) -> str:
|
|
984
|
+
"""Return the id of a collection by either its id or alias
|
|
948
985
|
|
|
949
|
-
:param alias_or_id: Alias of the
|
|
986
|
+
:param alias_or_id: Alias of the collection. If an existing id is given, this
|
|
950
987
|
method will directly return the given value.
|
|
951
|
-
:returns: Internal name of the
|
|
988
|
+
:returns: Internal name of the collection.
|
|
952
989
|
"""
|
|
953
|
-
|
|
954
|
-
k
|
|
955
|
-
for k, v in self.product_types_config.items()
|
|
956
|
-
if v.get("alias") == alias_or_id
|
|
990
|
+
collections = [
|
|
991
|
+
k for k, v in self.collections_config.items() if v.alias == alias_or_id
|
|
957
992
|
]
|
|
958
993
|
|
|
959
|
-
if len(
|
|
960
|
-
raise
|
|
961
|
-
f"Too many matching
|
|
994
|
+
if len(collections) > 1:
|
|
995
|
+
raise NoMatchingCollection(
|
|
996
|
+
f"Too many matching collections for alias {alias_or_id}: {collections}"
|
|
962
997
|
)
|
|
963
998
|
|
|
964
|
-
if len(
|
|
965
|
-
if alias_or_id in self.
|
|
999
|
+
if len(collections) == 0:
|
|
1000
|
+
if alias_or_id in self.collections_config:
|
|
966
1001
|
return alias_or_id
|
|
967
1002
|
else:
|
|
968
|
-
raise
|
|
969
|
-
f"Could not find
|
|
1003
|
+
raise NoMatchingCollection(
|
|
1004
|
+
f"Could not find collection from alias or id {alias_or_id}"
|
|
970
1005
|
)
|
|
971
1006
|
|
|
972
|
-
return
|
|
1007
|
+
return collections[0]
|
|
973
1008
|
|
|
974
|
-
def
|
|
975
|
-
"""Return the alias of a
|
|
976
|
-
given
|
|
1009
|
+
def get_alias_from_collection(self, collection: str) -> str:
|
|
1010
|
+
"""Return the alias of a collection by its id. If no alias was defined for the
|
|
1011
|
+
given collection, its id is returned instead.
|
|
977
1012
|
|
|
978
|
-
:param
|
|
979
|
-
:returns: Alias of the
|
|
1013
|
+
:param collection: collection id
|
|
1014
|
+
:returns: Alias of the collection or its id if no alias has been defined for it.
|
|
980
1015
|
"""
|
|
981
|
-
if
|
|
982
|
-
raise
|
|
1016
|
+
if collection not in self.collections_config:
|
|
1017
|
+
raise NoMatchingCollection(collection)
|
|
983
1018
|
|
|
984
|
-
|
|
1019
|
+
if alias := self.collections_config[collection].alias:
|
|
1020
|
+
return alias
|
|
1021
|
+
return collection
|
|
985
1022
|
|
|
986
|
-
def
|
|
1023
|
+
def guess_collection(
|
|
987
1024
|
self,
|
|
988
1025
|
free_text: Optional[str] = None,
|
|
989
1026
|
intersect: bool = False,
|
|
990
|
-
|
|
1027
|
+
instruments: Optional[str] = None,
|
|
991
1028
|
platform: Optional[str] = None,
|
|
992
|
-
|
|
993
|
-
|
|
994
|
-
|
|
1029
|
+
constellation: Optional[str] = None,
|
|
1030
|
+
processing_level: Optional[str] = None,
|
|
1031
|
+
sensor_type: Optional[str] = None,
|
|
995
1032
|
keywords: Optional[str] = None,
|
|
996
|
-
|
|
1033
|
+
description: Optional[str] = None,
|
|
997
1034
|
title: Optional[str] = None,
|
|
998
|
-
|
|
999
|
-
|
|
1035
|
+
start_date: Optional[str] = None,
|
|
1036
|
+
end_date: Optional[str] = None,
|
|
1000
1037
|
**kwargs: Any,
|
|
1001
|
-
) ->
|
|
1038
|
+
) -> CollectionsList:
|
|
1002
1039
|
"""
|
|
1003
|
-
Find EODAG
|
|
1040
|
+
Find EODAG collection IDs that best match a set of search parameters.
|
|
1004
1041
|
|
|
1005
|
-
When using several filters,
|
|
1042
|
+
When using several filters, collections that match most of them will be returned at first.
|
|
1006
1043
|
|
|
1007
1044
|
:param free_text: Free text search filter used to search accross all the following parameters. Handles logical
|
|
1008
1045
|
operators with parenthesis (``AND``/``OR``/``NOT``), quoted phrases (``"exact phrase"``),
|
|
1009
1046
|
``*`` and ``?`` wildcards.
|
|
1010
1047
|
:param intersect: Join results for each parameter using INTERSECT instead of UNION.
|
|
1011
|
-
:param
|
|
1048
|
+
:param instruments: Instruments parameter.
|
|
1012
1049
|
:param platform: Platform parameter.
|
|
1013
|
-
:param
|
|
1014
|
-
:param
|
|
1015
|
-
:param
|
|
1050
|
+
:param constellation: Constellation parameter.
|
|
1051
|
+
:param processing_level: Processing level parameter.
|
|
1052
|
+
:param sensor_type: Sensor type parameter.
|
|
1016
1053
|
:param keywords: Keywords parameter.
|
|
1017
|
-
:param
|
|
1054
|
+
:param description: description parameter.
|
|
1018
1055
|
:param title: Title parameter.
|
|
1019
|
-
:param
|
|
1020
|
-
:param
|
|
1056
|
+
:param start_date: start date for datetime filtering. Not used by free_text
|
|
1057
|
+
:param end_date: end date for datetime filtering. Not used by free_text
|
|
1021
1058
|
:returns: The best match for the given parameters.
|
|
1022
|
-
:raises: :class:`~eodag.utils.exceptions.
|
|
1059
|
+
:raises: :class:`~eodag.utils.exceptions.NoMatchingCollection`
|
|
1023
1060
|
"""
|
|
1024
|
-
if
|
|
1025
|
-
|
|
1061
|
+
if collection := kwargs.get("collection"):
|
|
1062
|
+
try:
|
|
1063
|
+
collection = self.get_collection_from_alias(collection)
|
|
1064
|
+
return CollectionsList([self.collections_config[collection]])
|
|
1065
|
+
except NoMatchingCollection:
|
|
1066
|
+
return CollectionsList(
|
|
1067
|
+
[Collection.create_with_dag(self, id=collection)]
|
|
1068
|
+
)
|
|
1026
1069
|
|
|
1027
1070
|
filters: dict[str, str] = {
|
|
1028
1071
|
k: v
|
|
1029
1072
|
for k, v in {
|
|
1030
|
-
"
|
|
1073
|
+
"instruments": instruments,
|
|
1074
|
+
"constellation": constellation,
|
|
1031
1075
|
"platform": platform,
|
|
1032
|
-
"
|
|
1033
|
-
"
|
|
1034
|
-
"sensorType": sensorType,
|
|
1076
|
+
"processing:level": processing_level,
|
|
1077
|
+
"eodag:sensor_type": sensor_type,
|
|
1035
1078
|
"keywords": keywords,
|
|
1036
|
-
"
|
|
1079
|
+
"description": description,
|
|
1037
1080
|
"title": title,
|
|
1038
1081
|
}.items()
|
|
1039
1082
|
if v is not None
|
|
@@ -1041,7 +1084,7 @@ class EODataAccessGateway:
|
|
|
1041
1084
|
|
|
1042
1085
|
only_dates = (
|
|
1043
1086
|
True
|
|
1044
|
-
if (not free_text and not filters and (
|
|
1087
|
+
if (not free_text and not filters and (start_date or end_date))
|
|
1045
1088
|
else False
|
|
1046
1089
|
)
|
|
1047
1090
|
|
|
@@ -1051,11 +1094,10 @@ class EODataAccessGateway:
|
|
|
1051
1094
|
|
|
1052
1095
|
guesses_with_score: list[tuple[str, int]] = []
|
|
1053
1096
|
|
|
1054
|
-
for
|
|
1097
|
+
for col, col_f in self.collections_config.items():
|
|
1055
1098
|
if (
|
|
1056
|
-
|
|
1057
|
-
or
|
|
1058
|
-
not in self._plugins_manager.product_type_to_provider_config_map
|
|
1099
|
+
col == GENERIC_COLLECTION
|
|
1100
|
+
or col not in self._plugins_manager.collection_to_provider_config_map
|
|
1059
1101
|
):
|
|
1060
1102
|
continue
|
|
1061
1103
|
|
|
@@ -1063,7 +1105,7 @@ class EODataAccessGateway:
|
|
|
1063
1105
|
|
|
1064
1106
|
# free text search
|
|
1065
1107
|
if free_text:
|
|
1066
|
-
match = free_text_evaluator(
|
|
1108
|
+
match = free_text_evaluator(col_f.model_dump())
|
|
1067
1109
|
if match:
|
|
1068
1110
|
score += 1
|
|
1069
1111
|
elif intersect:
|
|
@@ -1079,9 +1121,16 @@ class EODataAccessGateway:
|
|
|
1079
1121
|
}
|
|
1080
1122
|
|
|
1081
1123
|
filter_matches = [
|
|
1082
|
-
filters_evaluators[filter_name](
|
|
1124
|
+
filters_evaluators[filter_name](
|
|
1125
|
+
{
|
|
1126
|
+
filter_name: col_f.__dict__[
|
|
1127
|
+
Collection.get_collection_mtd_from_alias(filter_name)
|
|
1128
|
+
]
|
|
1129
|
+
}
|
|
1130
|
+
)
|
|
1083
1131
|
for filter_name, value in filters.items()
|
|
1084
|
-
if filter_name
|
|
1132
|
+
if Collection.get_collection_mtd_from_alias(filter_name)
|
|
1133
|
+
in col_f.__dict__
|
|
1085
1134
|
]
|
|
1086
1135
|
|
|
1087
1136
|
if filters_matching_method(filter_matches):
|
|
@@ -1094,43 +1143,39 @@ class EODataAccessGateway:
|
|
|
1094
1143
|
continue
|
|
1095
1144
|
|
|
1096
1145
|
# datetime filtering
|
|
1097
|
-
if
|
|
1146
|
+
if start_date or end_date:
|
|
1098
1147
|
min_aware = datetime.datetime.min.replace(tzinfo=datetime.timezone.utc)
|
|
1099
1148
|
max_aware = datetime.datetime.max.replace(tzinfo=datetime.timezone.utc)
|
|
1100
1149
|
|
|
1150
|
+
col_start = col_f.extent.temporal.interval[0][0]
|
|
1151
|
+
col_end = col_f.extent.temporal.interval[0][1]
|
|
1152
|
+
|
|
1101
1153
|
max_start = max(
|
|
1102
|
-
rfc3339_str_to_datetime(
|
|
1103
|
-
|
|
1104
|
-
else min_aware,
|
|
1105
|
-
rfc3339_str_to_datetime(pt_dict["missionStartDate"])
|
|
1106
|
-
if pt_dict.get("missionStartDate")
|
|
1107
|
-
else min_aware,
|
|
1154
|
+
rfc3339_str_to_datetime(start_date) if start_date else min_aware,
|
|
1155
|
+
col_start or min_aware,
|
|
1108
1156
|
)
|
|
1109
1157
|
min_end = min(
|
|
1110
|
-
rfc3339_str_to_datetime(
|
|
1111
|
-
|
|
1112
|
-
else max_aware,
|
|
1113
|
-
rfc3339_str_to_datetime(pt_dict["missionEndDate"])
|
|
1114
|
-
if pt_dict.get("missionEndDate")
|
|
1115
|
-
else max_aware,
|
|
1158
|
+
rfc3339_str_to_datetime(end_date) if end_date else max_aware,
|
|
1159
|
+
col_end or max_aware,
|
|
1116
1160
|
)
|
|
1117
1161
|
if not (max_start <= min_end):
|
|
1118
1162
|
continue
|
|
1119
1163
|
|
|
1120
|
-
|
|
1121
|
-
guesses_with_score.append((pt_alias, score))
|
|
1164
|
+
guesses_with_score.append((col_f._id, score))
|
|
1122
1165
|
|
|
1123
1166
|
if guesses_with_score:
|
|
1124
|
-
# sort by score descending, then
|
|
1167
|
+
# sort by score descending, then col for stability
|
|
1125
1168
|
guesses_with_score.sort(key=lambda x: (-x[1], x[0]))
|
|
1126
|
-
return
|
|
1169
|
+
return CollectionsList(
|
|
1170
|
+
[self.collections_config[col] for col, _ in guesses_with_score]
|
|
1171
|
+
)
|
|
1127
1172
|
|
|
1128
|
-
raise
|
|
1173
|
+
raise NoMatchingCollection()
|
|
1129
1174
|
|
|
1130
1175
|
def search(
|
|
1131
1176
|
self,
|
|
1132
1177
|
page: int = DEFAULT_PAGE,
|
|
1133
|
-
items_per_page: int = DEFAULT_ITEMS_PER_PAGE,
|
|
1178
|
+
items_per_page: Optional[int] = DEFAULT_ITEMS_PER_PAGE,
|
|
1134
1179
|
raise_errors: bool = False,
|
|
1135
1180
|
start: Optional[str] = None,
|
|
1136
1181
|
end: Optional[str] = None,
|
|
@@ -1138,20 +1183,22 @@ class EODataAccessGateway:
|
|
|
1138
1183
|
locations: Optional[dict[str, str]] = None,
|
|
1139
1184
|
provider: Optional[str] = None,
|
|
1140
1185
|
count: bool = False,
|
|
1186
|
+
validate: Optional[bool] = True,
|
|
1141
1187
|
**kwargs: Any,
|
|
1142
1188
|
) -> SearchResult:
|
|
1143
1189
|
"""Look for products matching criteria on known providers.
|
|
1144
1190
|
|
|
1145
1191
|
The default behaviour is to look for products on the provider with the
|
|
1146
|
-
highest priority supporting the requested
|
|
1192
|
+
highest priority supporting the requested collection. These priorities
|
|
1147
1193
|
are configurable through user configuration file or individual environment variable.
|
|
1148
1194
|
If the request to the provider with the highest priority fails or is empty, the data
|
|
1149
1195
|
will be request from the provider with the next highest priority.
|
|
1150
1196
|
Only if the request fails for all available providers, an error will be thrown.
|
|
1151
1197
|
|
|
1152
|
-
:param page: (optional) The page number to return
|
|
1198
|
+
:param page: (optional) The page number to return (**deprecated**, use
|
|
1199
|
+
:meth:`eodag.api.search_result.SearchResult.next_page` instead)
|
|
1153
1200
|
:param items_per_page: (optional) The number of results that must appear in one single
|
|
1154
|
-
page
|
|
1201
|
+
page. If ``None``, the maximum number possible will be used.
|
|
1155
1202
|
:param raise_errors: (optional) When an error occurs when searching, if this is set to
|
|
1156
1203
|
True, the error is raised
|
|
1157
1204
|
:param start: (optional) Start sensing time in ISO 8601 format (e.g. "1990-11-26",
|
|
@@ -1178,9 +1225,11 @@ class EODataAccessGateway:
|
|
|
1178
1225
|
If not set, the configured preferred provider will be used at first
|
|
1179
1226
|
before trying others until finding results.
|
|
1180
1227
|
:param count: (optional) Whether to run a query with a count request or not
|
|
1228
|
+
:param validate: (optional) Set to True to validate search parameters
|
|
1229
|
+
before sending the query to the provider
|
|
1181
1230
|
:param kwargs: Some other criteria that will be used to do the search,
|
|
1182
1231
|
using paramaters compatibles with the provider
|
|
1183
|
-
:returns: A
|
|
1232
|
+
:returns: A set of EO products matching the criteria
|
|
1184
1233
|
|
|
1185
1234
|
.. versionchanged:: v3.0.0b1
|
|
1186
1235
|
``search()`` method now returns only a single :class:`~eodag.api.search_result.SearchResult`
|
|
@@ -1191,6 +1240,15 @@ class EODataAccessGateway:
|
|
|
1191
1240
|
return a list as a result of their processing. This requirement is
|
|
1192
1241
|
enforced here.
|
|
1193
1242
|
"""
|
|
1243
|
+
if page != DEFAULT_PAGE:
|
|
1244
|
+
warnings.warn(
|
|
1245
|
+
"Usage of deprecated search parameter 'page' "
|
|
1246
|
+
"(Please use 'SearchResult.next_page()' instead)"
|
|
1247
|
+
" -- Deprecated since v3.9.0",
|
|
1248
|
+
DeprecationWarning,
|
|
1249
|
+
stacklevel=2,
|
|
1250
|
+
)
|
|
1251
|
+
|
|
1194
1252
|
search_plugins, search_kwargs = self._prepare_search(
|
|
1195
1253
|
start=start,
|
|
1196
1254
|
end=end,
|
|
@@ -1200,28 +1258,38 @@ class EODataAccessGateway:
|
|
|
1200
1258
|
**kwargs,
|
|
1201
1259
|
)
|
|
1202
1260
|
if search_kwargs.get("id"):
|
|
1261
|
+
# Don't validate requests by ID. "id" is not queryable.
|
|
1203
1262
|
return self._search_by_id(
|
|
1204
1263
|
search_kwargs.pop("id"),
|
|
1205
1264
|
provider=provider,
|
|
1206
1265
|
raise_errors=raise_errors,
|
|
1266
|
+
validate=False,
|
|
1207
1267
|
**search_kwargs,
|
|
1208
1268
|
)
|
|
1209
1269
|
# remove datacube query string from kwargs which was only needed for search-by-id
|
|
1210
1270
|
search_kwargs.pop("_dc_qs", None)
|
|
1211
|
-
|
|
1212
|
-
search_kwargs
|
|
1213
|
-
page=page,
|
|
1214
|
-
items_per_page=items_per_page,
|
|
1215
|
-
)
|
|
1271
|
+
# add page parameter
|
|
1272
|
+
search_kwargs["page"] = page
|
|
1216
1273
|
|
|
1217
1274
|
errors: list[tuple[str, Exception]] = []
|
|
1218
1275
|
# Loop over available providers and return the first non-empty results
|
|
1219
1276
|
for i, search_plugin in enumerate(search_plugins):
|
|
1220
1277
|
search_plugin.clear()
|
|
1278
|
+
|
|
1279
|
+
# add appropriate items_per_page value
|
|
1280
|
+
search_kwargs["items_per_page"] = (
|
|
1281
|
+
items_per_page
|
|
1282
|
+
if items_per_page is not None
|
|
1283
|
+
else getattr(search_plugin.config, "pagination", {}).get(
|
|
1284
|
+
"max_items_per_page", DEFAULT_MAX_ITEMS_PER_PAGE
|
|
1285
|
+
)
|
|
1286
|
+
)
|
|
1287
|
+
|
|
1221
1288
|
search_results = self._do_search(
|
|
1222
1289
|
search_plugin,
|
|
1223
1290
|
count=count,
|
|
1224
1291
|
raise_errors=raise_errors,
|
|
1292
|
+
validate=validate,
|
|
1225
1293
|
**search_kwargs,
|
|
1226
1294
|
)
|
|
1227
1295
|
errors.extend(search_results.errors)
|
|
@@ -1232,12 +1300,22 @@ class EODataAccessGateway:
|
|
|
1232
1300
|
)
|
|
1233
1301
|
elif len(search_results) > 0:
|
|
1234
1302
|
search_results.errors = errors
|
|
1303
|
+
if count and search_results.number_matched:
|
|
1304
|
+
logger.info(
|
|
1305
|
+
"Found %s result(s) on provider '%s'",
|
|
1306
|
+
search_results.number_matched,
|
|
1307
|
+
search_results[0].provider,
|
|
1308
|
+
)
|
|
1235
1309
|
return search_results
|
|
1236
1310
|
|
|
1237
1311
|
if i > 1:
|
|
1238
1312
|
logger.error("No result could be obtained from any available provider")
|
|
1239
1313
|
return SearchResult([], 0, errors) if count else SearchResult([], errors=errors)
|
|
1240
1314
|
|
|
1315
|
+
@_deprecated(
|
|
1316
|
+
reason="Please use 'SearchResult.next_page()' instead",
|
|
1317
|
+
version="v3.9.0",
|
|
1318
|
+
)
|
|
1241
1319
|
def search_iter_page(
|
|
1242
1320
|
self,
|
|
1243
1321
|
items_per_page: int = DEFAULT_ITEMS_PER_PAGE,
|
|
@@ -1249,6 +1327,9 @@ class EODataAccessGateway:
|
|
|
1249
1327
|
) -> Iterator[SearchResult]:
|
|
1250
1328
|
"""Iterate over the pages of a products search.
|
|
1251
1329
|
|
|
1330
|
+
.. deprecated:: v3.9.0
|
|
1331
|
+
Please use :meth:`eodag.api.search_result.SearchResult.next_page` instead.
|
|
1332
|
+
|
|
1252
1333
|
:param items_per_page: (optional) The number of results requested per page
|
|
1253
1334
|
:param start: (optional) Start sensing time in ISO 8601 format (e.g. "1990-11-26",
|
|
1254
1335
|
"1990-11-26T14:30:10.153Z", "1990-11-26T14:30:10+02:00", ...).
|
|
@@ -1272,7 +1353,7 @@ class EODataAccessGateway:
|
|
|
1272
1353
|
name=country and attr=ISO3
|
|
1273
1354
|
:param kwargs: Some other criteria that will be used to do the search,
|
|
1274
1355
|
using paramaters compatibles with the provider
|
|
1275
|
-
:returns: An iterator that yields page per page a
|
|
1356
|
+
:returns: An iterator that yields page per page a set of EO products
|
|
1276
1357
|
matching the criteria
|
|
1277
1358
|
"""
|
|
1278
1359
|
search_plugins, search_kwargs = self._prepare_search(
|
|
@@ -1299,6 +1380,10 @@ class EODataAccessGateway:
|
|
|
1299
1380
|
raise
|
|
1300
1381
|
raise RequestError("No result could be obtained from any available provider")
|
|
1301
1382
|
|
|
1383
|
+
@_deprecated(
|
|
1384
|
+
reason="Please use 'SearchResult.next_page()' instead",
|
|
1385
|
+
version="v3.9.0",
|
|
1386
|
+
)
|
|
1302
1387
|
def search_iter_page_plugin(
|
|
1303
1388
|
self,
|
|
1304
1389
|
search_plugin: Union[Search, Api],
|
|
@@ -1307,121 +1392,50 @@ class EODataAccessGateway:
|
|
|
1307
1392
|
) -> Iterator[SearchResult]:
|
|
1308
1393
|
"""Iterate over the pages of a products search using a given search plugin.
|
|
1309
1394
|
|
|
1395
|
+
.. deprecated:: v3.9.0
|
|
1396
|
+
Please use :meth:`eodag.api.search_result.SearchResult.next_page` instead.
|
|
1397
|
+
|
|
1310
1398
|
:param items_per_page: (optional) The number of results requested per page
|
|
1311
1399
|
:param kwargs: Some other criteria that will be used to do the search,
|
|
1312
1400
|
using parameters compatibles with the provider
|
|
1313
1401
|
:param search_plugin: search plugin to be used
|
|
1314
|
-
:returns: An iterator that yields page per page a
|
|
1402
|
+
:returns: An iterator that yields page per page a set of EO products
|
|
1315
1403
|
matching the criteria
|
|
1316
1404
|
"""
|
|
1317
|
-
|
|
1318
|
-
iteration = 1
|
|
1319
|
-
# Store the search plugin config pagination.next_page_url_tpl to reset it later
|
|
1320
|
-
# since it might be modified if the next_page_url mechanism is used by the
|
|
1321
|
-
# plugin. (same thing for next_page_query_obj, next_page_query_obj with POST reqs)
|
|
1322
|
-
pagination_config = getattr(search_plugin.config, "pagination", {})
|
|
1323
|
-
prev_next_page_url_tpl = pagination_config.get("next_page_url_tpl")
|
|
1324
|
-
prev_next_page_query_obj = pagination_config.get("next_page_query_obj")
|
|
1325
|
-
# Page has to be set to a value even if use_next is True, this is required
|
|
1326
|
-
# internally by the search plugin (see collect_search_urls)
|
|
1327
1405
|
kwargs.update(
|
|
1328
1406
|
page=1,
|
|
1329
1407
|
items_per_page=items_per_page,
|
|
1330
1408
|
)
|
|
1331
|
-
|
|
1332
|
-
|
|
1333
|
-
|
|
1334
|
-
|
|
1335
|
-
|
|
1336
|
-
|
|
1337
|
-
|
|
1338
|
-
|
|
1339
|
-
|
|
1340
|
-
|
|
1341
|
-
|
|
1342
|
-
|
|
1343
|
-
|
|
1344
|
-
|
|
1345
|
-
|
|
1346
|
-
|
|
1347
|
-
|
|
1348
|
-
|
|
1349
|
-
|
|
1350
|
-
|
|
1351
|
-
|
|
1352
|
-
|
|
1353
|
-
|
|
1354
|
-
|
|
1355
|
-
|
|
1356
|
-
|
|
1357
|
-
|
|
1358
|
-
|
|
1359
|
-
raise
|
|
1360
|
-
finally:
|
|
1361
|
-
# we don't want that next(search_iter_page(...)) modifies the plugin
|
|
1362
|
-
# indefinitely. So we reset after each request, but before the generator
|
|
1363
|
-
# yields, the attr next_page_url (to None) and
|
|
1364
|
-
# config.pagination["next_page_url_tpl"] (to its original value).
|
|
1365
|
-
next_page_url = getattr(search_plugin, "next_page_url", None)
|
|
1366
|
-
next_page_query_obj = getattr(search_plugin, "next_page_query_obj", {})
|
|
1367
|
-
next_page_merge = getattr(search_plugin, "next_page_merge", None)
|
|
1368
|
-
|
|
1369
|
-
if next_page_url:
|
|
1370
|
-
search_plugin.next_page_url = None
|
|
1371
|
-
if prev_next_page_url_tpl:
|
|
1372
|
-
search_plugin.config.pagination[
|
|
1373
|
-
"next_page_url_tpl"
|
|
1374
|
-
] = prev_next_page_url_tpl
|
|
1375
|
-
if next_page_query_obj:
|
|
1376
|
-
if prev_next_page_query_obj:
|
|
1377
|
-
search_plugin.config.pagination[
|
|
1378
|
-
"next_page_query_obj"
|
|
1379
|
-
] = prev_next_page_query_obj
|
|
1380
|
-
# Update next_page_query_obj for next page req
|
|
1381
|
-
if next_page_merge:
|
|
1382
|
-
search_plugin.next_page_query_obj = dict(
|
|
1383
|
-
getattr(search_plugin, "query_params", {}),
|
|
1384
|
-
**next_page_query_obj,
|
|
1385
|
-
)
|
|
1386
|
-
else:
|
|
1387
|
-
search_plugin.next_page_query_obj = next_page_query_obj
|
|
1388
|
-
|
|
1389
|
-
if len(search_result) > 0:
|
|
1390
|
-
# The first products between two iterations are compared. If they
|
|
1391
|
-
# are actually the same product, it means the iteration failed at
|
|
1392
|
-
# progressing for some reason. This is implemented as a workaround
|
|
1393
|
-
# to some search plugins/providers not handling pagination.
|
|
1394
|
-
product = search_result[0]
|
|
1395
|
-
if (
|
|
1396
|
-
prev_product
|
|
1397
|
-
and product.properties["id"] == prev_product.properties["id"]
|
|
1398
|
-
and product.provider == prev_product.provider
|
|
1399
|
-
):
|
|
1400
|
-
logger.warning(
|
|
1401
|
-
"Iterate over pages: stop iterating since the next page "
|
|
1402
|
-
"appears to have the same products as in the previous one. "
|
|
1403
|
-
"This provider may not implement pagination.",
|
|
1404
|
-
)
|
|
1405
|
-
last_page_with_products = iteration - 1
|
|
1406
|
-
break
|
|
1407
|
-
# use count got from 1st iteration
|
|
1408
|
-
search_result.number_matched = number_matched
|
|
1409
|
-
yield search_result
|
|
1410
|
-
prev_product = product
|
|
1411
|
-
# Prevent a last search if the current one returned less than the
|
|
1412
|
-
# maximum number of items asked for.
|
|
1413
|
-
if len(search_result) < items_per_page:
|
|
1414
|
-
last_page_with_products = iteration
|
|
1415
|
-
break
|
|
1416
|
-
else:
|
|
1417
|
-
last_page_with_products = iteration - 1
|
|
1409
|
+
try:
|
|
1410
|
+
# remove unwanted kwargs for _do_search
|
|
1411
|
+
kwargs.pop("raise_errors", None)
|
|
1412
|
+
search_result = self._do_search(search_plugin, raise_errors=True, **kwargs)
|
|
1413
|
+
search_result.raise_errors = True
|
|
1414
|
+
|
|
1415
|
+
except Exception:
|
|
1416
|
+
logger.warning(
|
|
1417
|
+
"error at retrieval of data from %s, for params: %s",
|
|
1418
|
+
search_plugin.provider,
|
|
1419
|
+
str(kwargs),
|
|
1420
|
+
)
|
|
1421
|
+
raise
|
|
1422
|
+
|
|
1423
|
+
if len(search_result) == 0:
|
|
1424
|
+
return
|
|
1425
|
+
# remove unwanted kwargs for next_page
|
|
1426
|
+
if kwargs.get("count") is True:
|
|
1427
|
+
kwargs["count"] = False
|
|
1428
|
+
kwargs.pop("page", None)
|
|
1429
|
+
search_result.search_params = kwargs
|
|
1430
|
+
if search_result._dag is None:
|
|
1431
|
+
search_result._dag = self
|
|
1432
|
+
|
|
1433
|
+
yield search_result
|
|
1434
|
+
|
|
1435
|
+
for next_result in search_result.next_page():
|
|
1436
|
+
if len(next_result) == 0:
|
|
1418
1437
|
break
|
|
1419
|
-
|
|
1420
|
-
kwargs["page"] = iteration
|
|
1421
|
-
logger.debug(
|
|
1422
|
-
"Iterate over pages: last products found on page %s",
|
|
1423
|
-
last_page_with_products,
|
|
1424
|
-
)
|
|
1438
|
+
yield next_result
|
|
1425
1439
|
|
|
1426
1440
|
def search_all(
|
|
1427
1441
|
self,
|
|
@@ -1471,84 +1485,45 @@ class EODataAccessGateway:
|
|
|
1471
1485
|
name=country and attr=ISO3
|
|
1472
1486
|
:param kwargs: Some other criteria that will be used to do the search,
|
|
1473
1487
|
using parameters compatible with the provider
|
|
1474
|
-
:returns: An iterator that yields page per page a
|
|
1488
|
+
:returns: An iterator that yields page per page a set of EO products
|
|
1475
1489
|
matching the criteria
|
|
1476
1490
|
"""
|
|
1477
|
-
# Get the search plugin and the maximized value
|
|
1478
|
-
# of items_per_page if defined for the provider used.
|
|
1479
|
-
try:
|
|
1480
|
-
product_type = self.get_product_type_from_alias(
|
|
1481
|
-
self.guess_product_type(**kwargs)[0]
|
|
1482
|
-
)
|
|
1483
|
-
except NoMatchingProductType:
|
|
1484
|
-
product_type = GENERIC_PRODUCT_TYPE
|
|
1485
|
-
else:
|
|
1486
|
-
# fetch product types list if product_type is unknown
|
|
1487
|
-
if (
|
|
1488
|
-
product_type
|
|
1489
|
-
not in self._plugins_manager.product_type_to_provider_config_map.keys()
|
|
1490
|
-
):
|
|
1491
|
-
logger.debug(
|
|
1492
|
-
f"Fetching external product types sources to find {product_type} product type"
|
|
1493
|
-
)
|
|
1494
|
-
self.fetch_product_types_list()
|
|
1495
|
-
|
|
1496
1491
|
# remove unwanted count
|
|
1497
1492
|
kwargs.pop("count", None)
|
|
1498
1493
|
|
|
1499
|
-
|
|
1500
|
-
|
|
1494
|
+
# First search
|
|
1495
|
+
search_results = self.search(
|
|
1496
|
+
items_per_page=items_per_page,
|
|
1497
|
+
start=start,
|
|
1498
|
+
end=end,
|
|
1499
|
+
geom=geom,
|
|
1500
|
+
locations=locations,
|
|
1501
|
+
**kwargs,
|
|
1501
1502
|
)
|
|
1502
|
-
|
|
1503
|
-
|
|
1504
|
-
|
|
1505
|
-
|
|
1506
|
-
|
|
1507
|
-
|
|
1508
|
-
|
|
1509
|
-
)
|
|
1503
|
+
if len(search_results) == 0:
|
|
1504
|
+
return search_results
|
|
1505
|
+
|
|
1506
|
+
try:
|
|
1507
|
+
search_results.raise_errors = True
|
|
1508
|
+
|
|
1509
|
+
# consume iterator
|
|
1510
|
+
deque(search_results.next_page(update=True))
|
|
1511
|
+
|
|
1510
1512
|
logger.info(
|
|
1511
|
-
"
|
|
1512
|
-
|
|
1513
|
-
|
|
1514
|
-
|
|
1513
|
+
"Found %s result(s) on provider '%s'",
|
|
1514
|
+
len(search_results),
|
|
1515
|
+
search_results[0].provider,
|
|
1516
|
+
)
|
|
1517
|
+
search_results.number_matched = len(search_results)
|
|
1518
|
+
except RequestError:
|
|
1519
|
+
logger.warning(
|
|
1520
|
+
"Found %s result(s) on provider '%s', but it may be incomplete "
|
|
1521
|
+
"as it ended with an error",
|
|
1522
|
+
len(search_results),
|
|
1523
|
+
search_results[0].provider,
|
|
1515
1524
|
)
|
|
1516
|
-
|
|
1517
|
-
|
|
1518
|
-
for page_results in self.search_iter_page_plugin(
|
|
1519
|
-
items_per_page=itp,
|
|
1520
|
-
search_plugin=search_plugin,
|
|
1521
|
-
count=False,
|
|
1522
|
-
**search_kwargs,
|
|
1523
|
-
):
|
|
1524
|
-
all_results.data.extend(page_results.data)
|
|
1525
|
-
logger.info(
|
|
1526
|
-
"Found %s result(s) on provider '%s'",
|
|
1527
|
-
len(all_results),
|
|
1528
|
-
search_plugin.provider,
|
|
1529
|
-
)
|
|
1530
|
-
return all_results
|
|
1531
|
-
except RequestError:
|
|
1532
|
-
if len(all_results) == 0 and i < len(search_plugins) - 1:
|
|
1533
|
-
logger.warning(
|
|
1534
|
-
"No result could be obtained from provider %s, "
|
|
1535
|
-
"we will try to get the data from another provider",
|
|
1536
|
-
search_plugin.provider,
|
|
1537
|
-
)
|
|
1538
|
-
elif len(all_results) == 0:
|
|
1539
|
-
logger.error(
|
|
1540
|
-
"No result could be obtained from any available provider"
|
|
1541
|
-
)
|
|
1542
|
-
raise
|
|
1543
|
-
elif len(all_results) > 0:
|
|
1544
|
-
logger.warning(
|
|
1545
|
-
"Found %s result(s) on provider '%s', but it may be incomplete "
|
|
1546
|
-
"as it ended with an error",
|
|
1547
|
-
len(all_results),
|
|
1548
|
-
search_plugin.provider,
|
|
1549
|
-
)
|
|
1550
|
-
return all_results
|
|
1551
|
-
raise RequestError("No result could be obtained from any available provider")
|
|
1525
|
+
|
|
1526
|
+
return search_results
|
|
1552
1527
|
|
|
1553
1528
|
def _search_by_id(
|
|
1554
1529
|
self, uid: str, provider: Optional[str] = None, **kwargs: Any
|
|
@@ -1571,13 +1546,13 @@ class EODataAccessGateway:
|
|
|
1571
1546
|
:param kwargs: Search criteria to help finding the right product
|
|
1572
1547
|
:returns: A search result with one EO product or None at all
|
|
1573
1548
|
"""
|
|
1574
|
-
|
|
1575
|
-
if
|
|
1549
|
+
collection = kwargs.get("collection")
|
|
1550
|
+
if collection is not None:
|
|
1576
1551
|
try:
|
|
1577
|
-
|
|
1578
|
-
except
|
|
1579
|
-
logger.debug("
|
|
1580
|
-
get_search_plugins_kwargs = dict(provider=provider,
|
|
1552
|
+
collection = self.get_collection_from_alias(collection)
|
|
1553
|
+
except NoMatchingCollection:
|
|
1554
|
+
logger.debug("collection %s not found", collection)
|
|
1555
|
+
get_search_plugins_kwargs = dict(provider=provider, collection=collection)
|
|
1581
1556
|
search_plugins = self._plugins_manager.get_search_plugins(
|
|
1582
1557
|
**get_search_plugins_kwargs
|
|
1583
1558
|
)
|
|
@@ -1632,10 +1607,10 @@ class EODataAccessGateway:
|
|
|
1632
1607
|
results = filtered
|
|
1633
1608
|
|
|
1634
1609
|
if len(results) == 1:
|
|
1635
|
-
if not results[0].
|
|
1636
|
-
# guess
|
|
1637
|
-
guesses = self.
|
|
1638
|
-
results[0].
|
|
1610
|
+
if not results[0].collection:
|
|
1611
|
+
# guess collection from properties
|
|
1612
|
+
guesses = self.guess_collection(**results[0].properties)
|
|
1613
|
+
results[0].collection = guesses[0].id
|
|
1639
1614
|
# reset driver
|
|
1640
1615
|
results[0].driver = results[0].get_driver()
|
|
1641
1616
|
results.number_matched = 1
|
|
@@ -1647,15 +1622,15 @@ class EODataAccessGateway:
|
|
|
1647
1622
|
)
|
|
1648
1623
|
return SearchResult([], 0, results.errors)
|
|
1649
1624
|
|
|
1650
|
-
def
|
|
1625
|
+
def _fetch_external_collection(self, provider: str, collection: str):
|
|
1651
1626
|
plugins = self._plugins_manager.get_search_plugins(provider=provider)
|
|
1652
1627
|
plugin = next(plugins)
|
|
1653
1628
|
|
|
1654
1629
|
# check after plugin init if still fetchable
|
|
1655
|
-
if not getattr(plugin.config, "
|
|
1630
|
+
if not getattr(plugin.config, "discover_collections", {}).get("fetch_url"):
|
|
1656
1631
|
return None
|
|
1657
1632
|
|
|
1658
|
-
kwargs: dict[str, Any] = {"
|
|
1633
|
+
kwargs: dict[str, Any] = {"collection": collection}
|
|
1659
1634
|
|
|
1660
1635
|
# append auth if needed
|
|
1661
1636
|
if getattr(plugin.config, "need_auth", False):
|
|
@@ -1666,8 +1641,8 @@ class EODataAccessGateway:
|
|
|
1666
1641
|
):
|
|
1667
1642
|
kwargs["auth"] = auth
|
|
1668
1643
|
|
|
1669
|
-
|
|
1670
|
-
self.
|
|
1644
|
+
collection_config = plugin.discover_collections(**kwargs)
|
|
1645
|
+
self.update_collections_list({provider: collection_config})
|
|
1671
1646
|
|
|
1672
1647
|
def _prepare_search(
|
|
1673
1648
|
self,
|
|
@@ -1683,9 +1658,9 @@ class EODataAccessGateway:
|
|
|
1683
1658
|
Product query:
|
|
1684
1659
|
* By id (plus optional 'provider')
|
|
1685
1660
|
* By search params:
|
|
1686
|
-
*
|
|
1687
|
-
* By
|
|
1688
|
-
* By params (e.g. 'platform'), see
|
|
1661
|
+
* collection query:
|
|
1662
|
+
* By collection (e.g. 'S2_MSI_L1C')
|
|
1663
|
+
* By params (e.g. 'platform'), see guess_collection
|
|
1689
1664
|
* dates: 'start' and/or 'end'
|
|
1690
1665
|
* geometry: 'geom' or 'bbox' or 'box'
|
|
1691
1666
|
* search locations
|
|
@@ -1700,53 +1675,53 @@ class EODataAccessGateway:
|
|
|
1700
1675
|
If no time offset is given, the time is assumed to be given in UTC.
|
|
1701
1676
|
:param geom: (optional) Search area that can be defined in different ways (see search)
|
|
1702
1677
|
:param locations: (optional) Location filtering by name using locations configuration
|
|
1703
|
-
:param provider: provider to be used, if no provider is given or the
|
|
1678
|
+
:param provider: provider to be used, if no provider is given or the collection
|
|
1704
1679
|
is not available for the provider, the preferred provider is used
|
|
1705
1680
|
:param kwargs: Some other criteria
|
|
1706
1681
|
* id and/or a provider for a search by
|
|
1707
|
-
* search criteria to guess the
|
|
1682
|
+
* search criteria to guess the collection
|
|
1708
1683
|
* other criteria compatible with the provider
|
|
1709
1684
|
:returns: Search plugins list and the prepared kwargs to make a query.
|
|
1710
1685
|
"""
|
|
1711
|
-
|
|
1712
|
-
if
|
|
1686
|
+
collection: Optional[str] = kwargs.get("collection")
|
|
1687
|
+
if collection is None:
|
|
1713
1688
|
try:
|
|
1714
|
-
guesses = self.
|
|
1689
|
+
guesses = self.guess_collection(**kwargs)
|
|
1715
1690
|
|
|
1716
|
-
#
|
|
1691
|
+
# guess_collection raises a NoMatchingCollection error if no product
|
|
1717
1692
|
# is found. Here, the supported search params are removed from the
|
|
1718
1693
|
# kwargs if present, not to propagate them to the query itself.
|
|
1719
1694
|
for param in (
|
|
1720
|
-
"
|
|
1695
|
+
"instruments",
|
|
1696
|
+
"constellation",
|
|
1721
1697
|
"platform",
|
|
1722
|
-
"
|
|
1723
|
-
"
|
|
1724
|
-
"sensorType",
|
|
1698
|
+
"processing:level",
|
|
1699
|
+
"eodag:sensor_type",
|
|
1725
1700
|
):
|
|
1726
1701
|
kwargs.pop(param, None)
|
|
1727
1702
|
|
|
1728
1703
|
# By now, only use the best bet
|
|
1729
|
-
|
|
1730
|
-
except
|
|
1704
|
+
collection = guesses[0].id
|
|
1705
|
+
except NoMatchingCollection:
|
|
1731
1706
|
queried_id = kwargs.get("id")
|
|
1732
1707
|
if queried_id is None:
|
|
1733
1708
|
logger.info(
|
|
1734
|
-
"No
|
|
1709
|
+
"No collection could be guessed with provided arguments"
|
|
1735
1710
|
)
|
|
1736
1711
|
else:
|
|
1737
1712
|
return [], kwargs
|
|
1738
1713
|
|
|
1739
|
-
if
|
|
1714
|
+
if collection is not None:
|
|
1740
1715
|
try:
|
|
1741
|
-
|
|
1742
|
-
except
|
|
1743
|
-
logger.info("unknown
|
|
1744
|
-
kwargs["
|
|
1716
|
+
collection = self.get_collection_from_alias(collection)
|
|
1717
|
+
except NoMatchingCollection:
|
|
1718
|
+
logger.info("unknown collection " + collection)
|
|
1719
|
+
kwargs["collection"] = collection
|
|
1745
1720
|
|
|
1746
1721
|
if start is not None:
|
|
1747
|
-
kwargs["
|
|
1722
|
+
kwargs["start_datetime"] = start
|
|
1748
1723
|
if end is not None:
|
|
1749
|
-
kwargs["
|
|
1724
|
+
kwargs["end_datetime"] = end
|
|
1750
1725
|
if "box" in kwargs or "bbox" in kwargs:
|
|
1751
1726
|
logger.warning(
|
|
1752
1727
|
"'box' or 'bbox' parameters are only supported for backwards "
|
|
@@ -1767,33 +1742,44 @@ class EODataAccessGateway:
|
|
|
1767
1742
|
kwargs.pop(arg, None)
|
|
1768
1743
|
del kwargs["locations"]
|
|
1769
1744
|
|
|
1770
|
-
# fetch
|
|
1745
|
+
# fetch collections list if collection is unknown
|
|
1771
1746
|
if (
|
|
1772
|
-
|
|
1773
|
-
not in self._plugins_manager.
|
|
1747
|
+
collection
|
|
1748
|
+
not in self._plugins_manager.collection_to_provider_config_map.keys()
|
|
1774
1749
|
):
|
|
1775
|
-
if provider and
|
|
1776
|
-
#
|
|
1777
|
-
logger.debug(
|
|
1778
|
-
|
|
1750
|
+
if provider and collection:
|
|
1751
|
+
# fetch ref for given provider and collection
|
|
1752
|
+
logger.debug(
|
|
1753
|
+
f"Fetching external collections sources to find {provider} {collection} collection"
|
|
1754
|
+
)
|
|
1755
|
+
self.fetch_collections_list(provider)
|
|
1756
|
+
if (
|
|
1757
|
+
collection
|
|
1758
|
+
not in self._plugins_manager.collection_to_provider_config_map.keys()
|
|
1759
|
+
):
|
|
1760
|
+
# Try to get specific collection from external provider
|
|
1761
|
+
logger.debug(
|
|
1762
|
+
"Fetching %s to find %s collection", provider, collection
|
|
1763
|
+
)
|
|
1764
|
+
self._fetch_external_collection(provider, collection)
|
|
1779
1765
|
if not provider:
|
|
1780
|
-
# no provider or still not found -> fetch all external
|
|
1766
|
+
# no provider or still not found -> fetch all external collections
|
|
1781
1767
|
logger.debug(
|
|
1782
|
-
f"Fetching external
|
|
1768
|
+
f"Fetching external collections sources to find {collection} collection"
|
|
1783
1769
|
)
|
|
1784
|
-
self.
|
|
1770
|
+
self.fetch_collections_list()
|
|
1785
1771
|
|
|
1786
1772
|
preferred_provider = self.get_preferred_provider()[0]
|
|
1787
1773
|
|
|
1788
1774
|
search_plugins: list[Union[Search, Api]] = []
|
|
1789
1775
|
for plugin in self._plugins_manager.get_search_plugins(
|
|
1790
|
-
|
|
1776
|
+
collection=collection, provider=provider
|
|
1791
1777
|
):
|
|
1792
|
-
# exclude MeteoblueSearch plugins from search fallback for unknown
|
|
1778
|
+
# exclude MeteoblueSearch plugins from search fallback for unknown collection
|
|
1793
1779
|
if (
|
|
1794
1780
|
provider != plugin.provider
|
|
1795
1781
|
and preferred_provider != plugin.provider
|
|
1796
|
-
and
|
|
1782
|
+
and collection not in self.collections_config
|
|
1797
1783
|
and isinstance(plugin, MeteoblueSearch)
|
|
1798
1784
|
):
|
|
1799
1785
|
continue
|
|
@@ -1804,8 +1790,8 @@ class EODataAccessGateway:
|
|
|
1804
1790
|
providers = [plugin.provider for plugin in search_plugins]
|
|
1805
1791
|
if provider not in providers:
|
|
1806
1792
|
logger.debug(
|
|
1807
|
-
"
|
|
1808
|
-
|
|
1793
|
+
"Collection '%s' is not available with preferred provider '%s'.",
|
|
1794
|
+
collection,
|
|
1809
1795
|
provider,
|
|
1810
1796
|
)
|
|
1811
1797
|
else:
|
|
@@ -1814,11 +1800,11 @@ class EODataAccessGateway:
|
|
|
1814
1800
|
)[0]
|
|
1815
1801
|
search_plugins.remove(provider_plugin)
|
|
1816
1802
|
search_plugins.insert(0, provider_plugin)
|
|
1817
|
-
# Add
|
|
1803
|
+
# Add collections_config to plugin config. This dict contains product
|
|
1818
1804
|
# type metadata that will also be stored in each product's properties.
|
|
1819
1805
|
for search_plugin in search_plugins:
|
|
1820
|
-
if
|
|
1821
|
-
self.
|
|
1806
|
+
if collection is not None:
|
|
1807
|
+
self._attach_collection_config(search_plugin, collection)
|
|
1822
1808
|
|
|
1823
1809
|
return search_plugins, kwargs
|
|
1824
1810
|
|
|
@@ -1827,6 +1813,7 @@ class EODataAccessGateway:
|
|
|
1827
1813
|
search_plugin: Union[Search, Api],
|
|
1828
1814
|
count: bool = False,
|
|
1829
1815
|
raise_errors: bool = False,
|
|
1816
|
+
validate: Optional[bool] = True,
|
|
1830
1817
|
**kwargs: Any,
|
|
1831
1818
|
) -> SearchResult:
|
|
1832
1819
|
"""Internal method that performs a search on a given provider.
|
|
@@ -1836,6 +1823,8 @@ class EODataAccessGateway:
|
|
|
1836
1823
|
:param raise_errors: (optional) When an error occurs when searching, if this is set to
|
|
1837
1824
|
True, the error is raised
|
|
1838
1825
|
:param kwargs: Some other criteria that will be used to do the search
|
|
1826
|
+
:param validate: (optional) Set to True to validate search parameters
|
|
1827
|
+
before sending the query to the provider
|
|
1839
1828
|
:returns: A collection of EO products matching the criteria
|
|
1840
1829
|
"""
|
|
1841
1830
|
logger.info("Searching on provider %s", search_plugin.provider)
|
|
@@ -1856,13 +1845,11 @@ class EODataAccessGateway:
|
|
|
1856
1845
|
max_items_per_page,
|
|
1857
1846
|
)
|
|
1858
1847
|
|
|
1859
|
-
results: list[EOProduct] = []
|
|
1860
|
-
total_results: Optional[int] = 0 if count else None
|
|
1861
|
-
|
|
1862
1848
|
errors: list[tuple[str, Exception]] = []
|
|
1863
1849
|
|
|
1864
1850
|
try:
|
|
1865
1851
|
prep = PreparedSearch(count=count)
|
|
1852
|
+
prep.raise_errors = raise_errors
|
|
1866
1853
|
|
|
1867
1854
|
# append auth if needed
|
|
1868
1855
|
if getattr(search_plugin.config, "need_auth", False):
|
|
@@ -1873,17 +1860,57 @@ class EODataAccessGateway:
|
|
|
1873
1860
|
):
|
|
1874
1861
|
prep.auth = auth
|
|
1875
1862
|
|
|
1876
|
-
prep.page = kwargs.pop("page", None)
|
|
1877
1863
|
prep.items_per_page = kwargs.pop("items_per_page", None)
|
|
1864
|
+
prep.next_page_token = kwargs.pop("next_page_token", None)
|
|
1865
|
+
prep.next_page_token_key = kwargs.pop(
|
|
1866
|
+
"next_page_token_key", None
|
|
1867
|
+
) or search_plugin.config.pagination.get("next_page_token_key", "page")
|
|
1868
|
+
prep.page = kwargs.pop("page", None)
|
|
1878
1869
|
|
|
1879
|
-
|
|
1870
|
+
if (
|
|
1871
|
+
prep.next_page_token_key == "page"
|
|
1872
|
+
and prep.items_per_page is not None
|
|
1873
|
+
and prep.next_page_token is None
|
|
1874
|
+
and prep.page is not None
|
|
1875
|
+
):
|
|
1876
|
+
prep.next_page_token = str(
|
|
1877
|
+
prep.page
|
|
1878
|
+
- 1
|
|
1879
|
+
+ search_plugin.config.pagination.get("start_page", DEFAULT_PAGE)
|
|
1880
|
+
)
|
|
1881
|
+
|
|
1882
|
+
# remove None values and convert param names to their pydantic alias if any
|
|
1883
|
+
search_params = {}
|
|
1884
|
+
ecmwf_queryables = [
|
|
1885
|
+
f"{ECMWF_PREFIX[:-1]}_{k}" for k in ECMWF_ALLOWED_KEYWORDS
|
|
1886
|
+
]
|
|
1887
|
+
for param, value in kwargs.items():
|
|
1888
|
+
if value is None:
|
|
1889
|
+
continue
|
|
1890
|
+
if param in Queryables.model_fields:
|
|
1891
|
+
param_alias = Queryables.model_fields[param].alias or param
|
|
1892
|
+
search_params[param_alias] = value
|
|
1893
|
+
elif param in ecmwf_queryables:
|
|
1894
|
+
# alias equivalent for ECMWF queryables
|
|
1895
|
+
search_params[
|
|
1896
|
+
re.sub(rf"^{ECMWF_PREFIX[:-1]}_", f"{ECMWF_PREFIX}", param)
|
|
1897
|
+
] = value
|
|
1898
|
+
else:
|
|
1899
|
+
# remove `provider:` or `provider_` prefix if any
|
|
1900
|
+
search_params[
|
|
1901
|
+
re.sub(r"^" + search_plugin.provider + r"[_:]", "", param)
|
|
1902
|
+
] = value
|
|
1903
|
+
|
|
1904
|
+
if validate:
|
|
1905
|
+
search_plugin.validate(search_params, prep.auth)
|
|
1880
1906
|
|
|
1881
|
-
|
|
1907
|
+
search_result = search_plugin.query(prep, **search_params)
|
|
1908
|
+
|
|
1909
|
+
if not isinstance(search_result.data, list):
|
|
1882
1910
|
raise PluginImplementationError(
|
|
1883
1911
|
"The query function of a Search plugin must return a list of "
|
|
1884
|
-
"results, got {} instead".format(type(
|
|
1912
|
+
"results, got {} instead".format(type(search_result.data))
|
|
1885
1913
|
)
|
|
1886
|
-
|
|
1887
1914
|
# Filter and attach to each eoproduct in the result the plugin capable of
|
|
1888
1915
|
# downloading it (this is done to enable the eo_product to download itself
|
|
1889
1916
|
# doing: eo_product.download()). The filtering is done by keeping only
|
|
@@ -1893,56 +1920,51 @@ class EODataAccessGateway:
|
|
|
1893
1920
|
# WARNING: this means an eo_product that has an invalid geometry can still
|
|
1894
1921
|
# be returned as a search result if there was no search extent (because we
|
|
1895
1922
|
# will not try to do an intersection)
|
|
1896
|
-
for eo_product in
|
|
1897
|
-
# if
|
|
1898
|
-
if eo_product.
|
|
1923
|
+
for eo_product in search_result.data:
|
|
1924
|
+
# if collection is not defined, try to guess using properties
|
|
1925
|
+
if eo_product.collection is None:
|
|
1899
1926
|
pattern = re.compile(r"[^\w,]+")
|
|
1900
1927
|
try:
|
|
1901
|
-
guesses = self.
|
|
1928
|
+
guesses = self.guess_collection(
|
|
1902
1929
|
intersect=False,
|
|
1903
1930
|
**{
|
|
1904
1931
|
k: pattern.sub("", str(v).upper())
|
|
1905
1932
|
for k, v in eo_product.properties.items()
|
|
1906
1933
|
if k
|
|
1907
1934
|
in [
|
|
1908
|
-
"
|
|
1935
|
+
"instruments",
|
|
1936
|
+
"constellation",
|
|
1909
1937
|
"platform",
|
|
1910
|
-
"
|
|
1911
|
-
"
|
|
1912
|
-
"sensorType",
|
|
1938
|
+
"processing:level",
|
|
1939
|
+
"eodag:sensor_type",
|
|
1913
1940
|
"keywords",
|
|
1914
1941
|
]
|
|
1915
1942
|
and v is not None
|
|
1916
1943
|
},
|
|
1917
1944
|
)
|
|
1918
|
-
except
|
|
1945
|
+
except NoMatchingCollection:
|
|
1919
1946
|
pass
|
|
1920
1947
|
else:
|
|
1921
|
-
eo_product.
|
|
1948
|
+
eo_product.collection = guesses[0].id
|
|
1922
1949
|
|
|
1923
1950
|
try:
|
|
1924
|
-
if eo_product.
|
|
1925
|
-
eo_product.
|
|
1926
|
-
eo_product.
|
|
1951
|
+
if eo_product.collection is not None:
|
|
1952
|
+
eo_product.collection = self.get_collection_from_alias(
|
|
1953
|
+
eo_product.collection
|
|
1927
1954
|
)
|
|
1928
|
-
except
|
|
1929
|
-
logger.debug("
|
|
1955
|
+
except NoMatchingCollection:
|
|
1956
|
+
logger.debug("collection %s not found", eo_product.collection)
|
|
1930
1957
|
|
|
1931
1958
|
if eo_product.search_intersection is not None:
|
|
1932
1959
|
eo_product._register_downloader_from_manager(self._plugins_manager)
|
|
1933
1960
|
|
|
1934
|
-
|
|
1935
|
-
|
|
1936
|
-
None
|
|
1937
|
-
|
|
1938
|
-
|
|
1939
|
-
|
|
1940
|
-
|
|
1941
|
-
logger.info(
|
|
1942
|
-
"Found %s result(s) on provider '%s'",
|
|
1943
|
-
nb_res,
|
|
1944
|
-
search_plugin.provider,
|
|
1945
|
-
)
|
|
1961
|
+
# Make next_page not available if the current one returned less than the maximum number of items asked for.
|
|
1962
|
+
if not prep.items_per_page or len(search_result) < prep.items_per_page:
|
|
1963
|
+
search_result.next_page_token = None
|
|
1964
|
+
|
|
1965
|
+
search_result._dag = self
|
|
1966
|
+
return search_result
|
|
1967
|
+
|
|
1946
1968
|
except Exception as e:
|
|
1947
1969
|
if raise_errors:
|
|
1948
1970
|
# Raise the error, letting the application wrapping eodag know that
|
|
@@ -1954,7 +1976,7 @@ class EODataAccessGateway:
|
|
|
1954
1976
|
search_plugin.provider,
|
|
1955
1977
|
)
|
|
1956
1978
|
errors.append((search_plugin.provider, e))
|
|
1957
|
-
|
|
1979
|
+
return SearchResult([], 0, errors)
|
|
1958
1980
|
|
|
1959
1981
|
def crunch(self, results: SearchResult, **kwargs: Any) -> SearchResult:
|
|
1960
1982
|
"""Apply the filters given through the keyword arguments to the results
|
|
@@ -2005,7 +2027,7 @@ class EODataAccessGateway:
|
|
|
2005
2027
|
) -> list[str]:
|
|
2006
2028
|
"""Download all products resulting from a search.
|
|
2007
2029
|
|
|
2008
|
-
:param search_result: A
|
|
2030
|
+
:param search_result: A set of EO products resulting from a search
|
|
2009
2031
|
:param downloaded_callback: (optional) A method or a callable object which takes
|
|
2010
2032
|
as parameter the ``product``. You can use the base class
|
|
2011
2033
|
:class:`~eodag.utils.DownloadedCallback` and override
|
|
@@ -2058,13 +2080,16 @@ class EODataAccessGateway:
|
|
|
2058
2080
|
search_result: SearchResult, filename: str = "search_results.geojson"
|
|
2059
2081
|
) -> str:
|
|
2060
2082
|
"""Registers results of a search into a geojson file.
|
|
2083
|
+
The output is a FeatureCollection containing the EO products as features,
|
|
2084
|
+
with additional metadata such as ``number_matched``, ``next_page_token``,
|
|
2085
|
+
and ``search_params`` stored in the properties.
|
|
2061
2086
|
|
|
2062
|
-
:param search_result: A
|
|
2087
|
+
:param search_result: A set of EO products resulting from a search
|
|
2063
2088
|
:param filename: (optional) The name of the file to generate
|
|
2064
2089
|
:returns: The name of the created file
|
|
2065
2090
|
"""
|
|
2066
2091
|
with open(filename, "w") as fh:
|
|
2067
|
-
geojson.dump(search_result, fh)
|
|
2092
|
+
geojson.dump(search_result.as_geojson_object(), fh)
|
|
2068
2093
|
return filename
|
|
2069
2094
|
|
|
2070
2095
|
@staticmethod
|
|
@@ -2079,12 +2104,16 @@ class EODataAccessGateway:
|
|
|
2079
2104
|
|
|
2080
2105
|
def deserialize_and_register(self, filename: str) -> SearchResult:
|
|
2081
2106
|
"""Loads results of a search from a geojson file and register
|
|
2082
|
-
products with the information needed to download itself
|
|
2107
|
+
products with the information needed to download itself.
|
|
2108
|
+
|
|
2109
|
+
This method also sets the internal EODataAccessGateway instance on the products,
|
|
2110
|
+
enabling pagination (e.g. access to next pages) if available.
|
|
2083
2111
|
|
|
2084
2112
|
:param filename: A filename containing a search result encoded as a geojson
|
|
2085
|
-
:returns: The search results encoded in `filename
|
|
2113
|
+
:returns: The search results encoded in `filename`, ready for download and pagination
|
|
2086
2114
|
"""
|
|
2087
2115
|
products = self.deserialize(filename)
|
|
2116
|
+
products._dag = self
|
|
2088
2117
|
for i, product in enumerate(products):
|
|
2089
2118
|
if product.downloader is None:
|
|
2090
2119
|
downloader = self._plugins_manager.get_download_plugin(product)
|
|
@@ -2095,69 +2124,6 @@ class EODataAccessGateway:
|
|
|
2095
2124
|
|
|
2096
2125
|
return products
|
|
2097
2126
|
|
|
2098
|
-
@_deprecated(
|
|
2099
|
-
reason="Use the StaticStacSearch search plugin instead", version="2.2.1"
|
|
2100
|
-
)
|
|
2101
|
-
def load_stac_items(
|
|
2102
|
-
self,
|
|
2103
|
-
filename: str,
|
|
2104
|
-
recursive: bool = False,
|
|
2105
|
-
max_connections: int = 100,
|
|
2106
|
-
provider: Optional[str] = None,
|
|
2107
|
-
productType: Optional[str] = None,
|
|
2108
|
-
timeout: int = HTTP_REQ_TIMEOUT,
|
|
2109
|
-
ssl_verify: bool = True,
|
|
2110
|
-
**kwargs: Any,
|
|
2111
|
-
) -> SearchResult:
|
|
2112
|
-
"""Loads STAC items from a geojson file / STAC catalog or collection, and convert to SearchResult.
|
|
2113
|
-
|
|
2114
|
-
Features are parsed using eodag provider configuration, as if they were
|
|
2115
|
-
the response content to an API request.
|
|
2116
|
-
|
|
2117
|
-
:param filename: A filename containing features encoded as a geojson
|
|
2118
|
-
:param recursive: (optional) Browse recursively in child nodes if True
|
|
2119
|
-
:param max_connections: (optional) Maximum number of connections for concurrent HTTP requests
|
|
2120
|
-
:param provider: (optional) Data provider
|
|
2121
|
-
:param productType: (optional) Data product type
|
|
2122
|
-
:param timeout: (optional) Timeout in seconds for each internal HTTP request
|
|
2123
|
-
:param kwargs: Parameters that will be stored in the result as
|
|
2124
|
-
search criteria
|
|
2125
|
-
:returns: The search results encoded in `filename`
|
|
2126
|
-
|
|
2127
|
-
.. deprecated:: 2.2.1
|
|
2128
|
-
Use the :class:`~eodag.plugins.search.static_stac_search.StaticStacSearch` search plugin instead.
|
|
2129
|
-
"""
|
|
2130
|
-
features = fetch_stac_items(
|
|
2131
|
-
filename,
|
|
2132
|
-
recursive=recursive,
|
|
2133
|
-
max_connections=max_connections,
|
|
2134
|
-
timeout=timeout,
|
|
2135
|
-
ssl_verify=ssl_verify,
|
|
2136
|
-
)
|
|
2137
|
-
feature_collection = geojson.FeatureCollection(features)
|
|
2138
|
-
|
|
2139
|
-
plugin = next(
|
|
2140
|
-
self._plugins_manager.get_search_plugins(
|
|
2141
|
-
product_type=productType, provider=provider
|
|
2142
|
-
)
|
|
2143
|
-
)
|
|
2144
|
-
# save plugin._request and mock it to make return loaded static results
|
|
2145
|
-
plugin_request = plugin._request
|
|
2146
|
-
plugin._request = (
|
|
2147
|
-
lambda url, info_message=None, exception_message=None: MockResponse(
|
|
2148
|
-
feature_collection, 200
|
|
2149
|
-
)
|
|
2150
|
-
)
|
|
2151
|
-
|
|
2152
|
-
search_result = self.search(
|
|
2153
|
-
productType=productType, provider=provider, **kwargs
|
|
2154
|
-
)
|
|
2155
|
-
|
|
2156
|
-
# restore plugin._request
|
|
2157
|
-
plugin._request = plugin_request
|
|
2158
|
-
|
|
2159
|
-
return search_result
|
|
2160
|
-
|
|
2161
2127
|
def download(
|
|
2162
2128
|
self,
|
|
2163
2129
|
product: EOProduct,
|
|
@@ -2173,16 +2139,16 @@ class EODataAccessGateway:
|
|
|
2173
2139
|
checks like verifying that a downloader and authenticator are registered
|
|
2174
2140
|
for the product before trying to download it.
|
|
2175
2141
|
|
|
2176
|
-
If the metadata mapping for ``
|
|
2142
|
+
If the metadata mapping for ``eodag:download_link`` is set to something that can be
|
|
2177
2143
|
interpreted as a link on a
|
|
2178
2144
|
local filesystem, the download is skipped (by now, only a link starting
|
|
2179
2145
|
with ``file:/`` is supported). Therefore, any user that knows how to extract
|
|
2180
2146
|
product location from product metadata on a provider can override the
|
|
2181
|
-
``
|
|
2147
|
+
``eodag:download_link`` metadata mapping in the right way. For example, using the
|
|
2182
2148
|
environment variable:
|
|
2183
|
-
``
|
|
2149
|
+
``EODAG__CREODIAS__SEARCH__METADATA_MAPPING__EODAG_DOWNLOAD_LINK="file:///{id}"`` will
|
|
2184
2150
|
lead to all :class:`~eodag.api.product._product.EOProduct`'s originating from the
|
|
2185
|
-
provider ``creodias`` to have their ``
|
|
2151
|
+
provider ``creodias`` to have their ``eodag:download_link`` metadata point to something like:
|
|
2186
2152
|
``file:///12345-678``, making this method immediately return the later string without
|
|
2187
2153
|
trying to download the product.
|
|
2188
2154
|
|
|
@@ -2246,46 +2212,46 @@ class EODataAccessGateway:
|
|
|
2246
2212
|
fetch_providers: bool = True,
|
|
2247
2213
|
**kwargs: Any,
|
|
2248
2214
|
) -> QueryablesDict:
|
|
2249
|
-
"""Fetch the queryable properties for a given
|
|
2215
|
+
"""Fetch the queryable properties for a given collection and/or provider.
|
|
2250
2216
|
|
|
2251
2217
|
:param provider: (optional) The provider.
|
|
2252
|
-
:param fetch_providers: If new
|
|
2253
|
-
:param kwargs: additional filters for queryables (`
|
|
2218
|
+
:param fetch_providers: If new collections should be fetched from the providers; default: True
|
|
2219
|
+
:param kwargs: additional filters for queryables (`collection` or other search
|
|
2254
2220
|
arguments)
|
|
2255
2221
|
|
|
2256
|
-
:raises
|
|
2222
|
+
:raises UnsupportedCollection: If the specified collection is not available for the
|
|
2257
2223
|
provider.
|
|
2258
2224
|
|
|
2259
2225
|
:returns: A :class:`~eodag.api.product.queryables.QuerybalesDict` containing the EODAG queryable
|
|
2260
2226
|
properties, associating parameters to their annotated type, and a additional_properties attribute
|
|
2261
2227
|
"""
|
|
2262
|
-
# only fetch providers if
|
|
2263
|
-
|
|
2264
|
-
|
|
2265
|
-
for
|
|
2228
|
+
# only fetch providers if collection is not found
|
|
2229
|
+
available_collections: list[str] = [
|
|
2230
|
+
col.id
|
|
2231
|
+
for col in self.list_collections(provider=provider, fetch_providers=False)
|
|
2266
2232
|
]
|
|
2267
|
-
|
|
2268
|
-
|
|
2233
|
+
collection: Optional[str] = kwargs.get("collection")
|
|
2234
|
+
coll_alias: Optional[str] = collection
|
|
2269
2235
|
|
|
2270
|
-
if
|
|
2271
|
-
if
|
|
2236
|
+
if collection:
|
|
2237
|
+
if collection not in available_collections:
|
|
2272
2238
|
if fetch_providers:
|
|
2273
2239
|
# fetch providers and try again
|
|
2274
|
-
|
|
2275
|
-
|
|
2276
|
-
for
|
|
2240
|
+
available_collections = [
|
|
2241
|
+
col.id
|
|
2242
|
+
for col in self.list_collections(
|
|
2277
2243
|
provider=provider, fetch_providers=True
|
|
2278
2244
|
)
|
|
2279
2245
|
]
|
|
2280
|
-
raise
|
|
2246
|
+
raise UnsupportedCollection(f"{collection} is not available.")
|
|
2281
2247
|
try:
|
|
2282
|
-
kwargs["
|
|
2283
|
-
|
|
2248
|
+
kwargs["collection"] = collection = self.get_collection_from_alias(
|
|
2249
|
+
collection
|
|
2284
2250
|
)
|
|
2285
|
-
except
|
|
2286
|
-
raise
|
|
2251
|
+
except NoMatchingCollection as e:
|
|
2252
|
+
raise UnsupportedCollection(f"{collection} is not available.") from e
|
|
2287
2253
|
|
|
2288
|
-
if not provider and not
|
|
2254
|
+
if not provider and not collection:
|
|
2289
2255
|
return QueryablesDict(
|
|
2290
2256
|
additional_properties=True,
|
|
2291
2257
|
**model_fields_to_annotated(CommonQueryables.model_fields),
|
|
@@ -2295,16 +2261,16 @@ class EODataAccessGateway:
|
|
|
2295
2261
|
additional_information = []
|
|
2296
2262
|
queryable_properties: dict[str, Any] = {}
|
|
2297
2263
|
|
|
2298
|
-
for plugin in self._plugins_manager.get_search_plugins(
|
|
2299
|
-
# attach
|
|
2300
|
-
|
|
2301
|
-
if
|
|
2302
|
-
self.
|
|
2303
|
-
|
|
2264
|
+
for plugin in self._plugins_manager.get_search_plugins(collection, provider):
|
|
2265
|
+
# attach collection config
|
|
2266
|
+
collection_configs: dict[str, Any] = {}
|
|
2267
|
+
if collection:
|
|
2268
|
+
self._attach_collection_config(plugin, collection)
|
|
2269
|
+
collection_configs[collection] = plugin.config.collection_config
|
|
2304
2270
|
else:
|
|
2305
|
-
for
|
|
2306
|
-
self.
|
|
2307
|
-
|
|
2271
|
+
for col in available_collections:
|
|
2272
|
+
self._attach_collection_config(plugin, col)
|
|
2273
|
+
collection_configs[col] = plugin.config.collection_config
|
|
2308
2274
|
|
|
2309
2275
|
# authenticate if required
|
|
2310
2276
|
if getattr(plugin.config, "need_auth", False) and (
|
|
@@ -2326,10 +2292,10 @@ class EODataAccessGateway:
|
|
|
2326
2292
|
|
|
2327
2293
|
plugin_queryables = plugin.list_queryables(
|
|
2328
2294
|
kwargs_alias,
|
|
2329
|
-
|
|
2330
|
-
|
|
2331
|
-
|
|
2332
|
-
|
|
2295
|
+
available_collections,
|
|
2296
|
+
collection_configs,
|
|
2297
|
+
collection,
|
|
2298
|
+
coll_alias,
|
|
2333
2299
|
)
|
|
2334
2300
|
|
|
2335
2301
|
if plugin_queryables.additional_information:
|
|
@@ -2379,32 +2345,31 @@ class EODataAccessGateway:
|
|
|
2379
2345
|
}
|
|
2380
2346
|
return sortables
|
|
2381
2347
|
|
|
2382
|
-
def
|
|
2348
|
+
def _attach_collection_config(self, plugin: Search, collection: str) -> None:
|
|
2383
2349
|
"""
|
|
2384
|
-
Attach
|
|
2350
|
+
Attach collections_config to plugin config. This dict contains product
|
|
2385
2351
|
type metadata that will also be stored in each product's properties.
|
|
2386
2352
|
"""
|
|
2387
2353
|
try:
|
|
2388
|
-
plugin.config.
|
|
2354
|
+
plugin.config.collection_config = dict(
|
|
2389
2355
|
[
|
|
2390
|
-
|
|
2391
|
-
for
|
|
2356
|
+
c.model_dump(mode="json", exclude={"id"})
|
|
2357
|
+
for c in self.list_collections(
|
|
2392
2358
|
plugin.provider, fetch_providers=False
|
|
2393
2359
|
)
|
|
2394
|
-
if
|
|
2360
|
+
if c._id == collection
|
|
2395
2361
|
][0],
|
|
2396
|
-
**{"
|
|
2362
|
+
**{"collection": collection},
|
|
2397
2363
|
)
|
|
2398
|
-
# If the product isn't in the catalog, it's a generic
|
|
2364
|
+
# If the product isn't in the catalog, it's a generic collection.
|
|
2399
2365
|
except IndexError:
|
|
2400
|
-
# Construct the
|
|
2401
|
-
plugin.config.
|
|
2402
|
-
|
|
2403
|
-
|
|
2404
|
-
|
|
2366
|
+
# Construct the GENERIC_COLLECTION metadata
|
|
2367
|
+
plugin.config.collection_config = dict(
|
|
2368
|
+
**self.collections_config[GENERIC_COLLECTION].model_dump(
|
|
2369
|
+
mode="json", exclude={"id"}
|
|
2370
|
+
),
|
|
2371
|
+
collection=collection,
|
|
2405
2372
|
)
|
|
2406
|
-
# Remove the ID since this is equal to productType.
|
|
2407
|
-
plugin.config.product_type_config.pop("ID", None)
|
|
2408
2373
|
|
|
2409
2374
|
def import_stac_items(self, items_urls: list[str]) -> SearchResult:
|
|
2410
2375
|
"""Import STAC items from a list of URLs and convert them to SearchResult.
|