eodag 3.9.1__py3-none-any.whl → 4.0.0a1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- eodag/api/core.py +378 -419
- eodag/api/product/__init__.py +3 -3
- eodag/api/product/_product.py +68 -40
- eodag/api/product/drivers/__init__.py +3 -5
- eodag/api/product/drivers/base.py +1 -18
- eodag/api/product/metadata_mapping.py +151 -215
- eodag/api/search_result.py +13 -7
- eodag/cli.py +72 -395
- eodag/config.py +46 -50
- eodag/plugins/apis/base.py +2 -2
- eodag/plugins/apis/ecmwf.py +20 -21
- eodag/plugins/apis/usgs.py +37 -33
- eodag/plugins/authentication/aws_auth.py +36 -1
- eodag/plugins/authentication/base.py +18 -3
- eodag/plugins/authentication/sas_auth.py +15 -0
- eodag/plugins/crunch/filter_date.py +3 -3
- eodag/plugins/crunch/filter_latest_intersect.py +2 -2
- eodag/plugins/crunch/filter_latest_tpl_name.py +1 -1
- eodag/plugins/download/aws.py +45 -41
- eodag/plugins/download/base.py +13 -14
- eodag/plugins/download/http.py +65 -65
- eodag/plugins/manager.py +28 -29
- eodag/plugins/search/__init__.py +3 -4
- eodag/plugins/search/base.py +128 -77
- eodag/plugins/search/build_search_result.py +105 -107
- eodag/plugins/search/cop_marine.py +44 -47
- eodag/plugins/search/csw.py +33 -33
- eodag/plugins/search/qssearch.py +335 -354
- eodag/plugins/search/stac_list_assets.py +1 -1
- eodag/plugins/search/static_stac_search.py +31 -31
- eodag/resources/{product_types.yml → collections.yml} +2353 -2429
- eodag/resources/ext_collections.json +1 -0
- eodag/resources/ext_product_types.json +1 -1
- eodag/resources/providers.yml +2432 -2714
- eodag/resources/stac_provider.yml +46 -90
- eodag/types/queryables.py +55 -91
- eodag/types/search_args.py +1 -1
- eodag/utils/__init__.py +94 -21
- eodag/utils/exceptions.py +6 -6
- eodag/utils/free_text_search.py +3 -3
- {eodag-3.9.1.dist-info → eodag-4.0.0a1.dist-info}/METADATA +11 -88
- eodag-4.0.0a1.dist-info/RECORD +92 -0
- {eodag-3.9.1.dist-info → eodag-4.0.0a1.dist-info}/entry_points.txt +0 -4
- eodag/plugins/authentication/oauth.py +0 -60
- eodag/plugins/download/creodias_s3.py +0 -64
- eodag/plugins/download/s3rest.py +0 -351
- eodag/plugins/search/data_request_search.py +0 -565
- eodag/resources/stac.yml +0 -294
- eodag/resources/stac_api.yml +0 -2105
- eodag/rest/__init__.py +0 -24
- eodag/rest/cache.py +0 -70
- eodag/rest/config.py +0 -67
- eodag/rest/constants.py +0 -26
- eodag/rest/core.py +0 -764
- eodag/rest/errors.py +0 -210
- eodag/rest/server.py +0 -604
- eodag/rest/server.wsgi +0 -6
- eodag/rest/stac.py +0 -1032
- eodag/rest/templates/README +0 -1
- eodag/rest/types/__init__.py +0 -18
- eodag/rest/types/collections_search.py +0 -44
- eodag/rest/types/eodag_search.py +0 -386
- eodag/rest/types/queryables.py +0 -174
- eodag/rest/types/stac_search.py +0 -272
- eodag/rest/utils/__init__.py +0 -207
- eodag/rest/utils/cql_evaluate.py +0 -119
- eodag/rest/utils/rfc3339.py +0 -64
- eodag-3.9.1.dist-info/RECORD +0 -115
- {eodag-3.9.1.dist-info → eodag-4.0.0a1.dist-info}/WHEEL +0 -0
- {eodag-3.9.1.dist-info → eodag-4.0.0a1.dist-info}/licenses/LICENSE +0 -0
- {eodag-3.9.1.dist-info → eodag-4.0.0a1.dist-info}/top_level.txt +0 -0
eodag/plugins/search/qssearch.py
CHANGED
|
@@ -76,17 +76,17 @@ from eodag.types.queryables import Queryables
|
|
|
76
76
|
from eodag.types.search_args import SortByList
|
|
77
77
|
from eodag.utils import (
|
|
78
78
|
DEFAULT_SEARCH_TIMEOUT,
|
|
79
|
-
|
|
79
|
+
GENERIC_COLLECTION,
|
|
80
80
|
HTTP_REQ_TIMEOUT,
|
|
81
81
|
REQ_RETRY_BACKOFF_FACTOR,
|
|
82
82
|
REQ_RETRY_STATUS_FORCELIST,
|
|
83
83
|
REQ_RETRY_TOTAL,
|
|
84
84
|
USER_AGENT,
|
|
85
|
-
_deprecated,
|
|
86
85
|
copy_deepcopy,
|
|
87
86
|
deepcopy,
|
|
88
87
|
dict_items_recursive_apply,
|
|
89
88
|
format_dict_items,
|
|
89
|
+
format_string,
|
|
90
90
|
get_ssl_context,
|
|
91
91
|
string_to_jsonpath,
|
|
92
92
|
update_nested_dict,
|
|
@@ -167,38 +167,38 @@ class QueryStringSearch(Search):
|
|
|
167
167
|
page that the provider can handle; default: ``50``
|
|
168
168
|
* :attr:`~eodag.config.PluginConfig.Pagination.start_page` (``int``): number of the first page; default: ``1``
|
|
169
169
|
|
|
170
|
-
* :attr:`~eodag.config.PluginConfig.
|
|
171
|
-
(:class:`~eodag.config.PluginConfig.
|
|
170
|
+
* :attr:`~eodag.config.PluginConfig.discover_collections`
|
|
171
|
+
(:class:`~eodag.config.PluginConfig.DiscoverCollections`): configuration for collection discovery based on
|
|
172
172
|
information from the provider; It contains the keys:
|
|
173
173
|
|
|
174
|
-
* :attr:`~eodag.config.PluginConfig.
|
|
175
|
-
the
|
|
176
|
-
* :attr:`~eodag.config.PluginConfig.
|
|
174
|
+
* :attr:`~eodag.config.PluginConfig.DiscoverCollections.fetch_url` (``str``) (**mandatory**): url from which
|
|
175
|
+
the collections can be fetched
|
|
176
|
+
* :attr:`~eodag.config.PluginConfig.DiscoverCollections.max_connections` (``int``): Maximum number of
|
|
177
177
|
connections for concurrent HTTP requests
|
|
178
|
-
* :attr:`~eodag.config.PluginConfig.
|
|
178
|
+
* :attr:`~eodag.config.PluginConfig.DiscoverCollections.result_type` (``str``): type of the provider result;
|
|
179
179
|
currently only ``json`` is supported (other types could be used in an extension of this plugin)
|
|
180
|
-
* :attr:`~eodag.config.PluginConfig.
|
|
181
|
-
to the list of
|
|
182
|
-
* :attr:`~eodag.config.PluginConfig.
|
|
183
|
-
|
|
184
|
-
* :attr:`~eodag.config.PluginConfig.
|
|
185
|
-
(``dict[str, str]``): mapping for
|
|
180
|
+
* :attr:`~eodag.config.PluginConfig.DiscoverCollections.results_entry` (``str``) (**mandatory**): json path
|
|
181
|
+
to the list of collections
|
|
182
|
+
* :attr:`~eodag.config.PluginConfig.DiscoverCollections.generic_collection_id` (``str``): mapping for the
|
|
183
|
+
collection id
|
|
184
|
+
* :attr:`~eodag.config.PluginConfig.DiscoverCollections.generic_collection_parsable_metadata`
|
|
185
|
+
(``dict[str, str]``): mapping for collection metadata (e.g. ``abstract``, ``licence``) which can be parsed
|
|
186
186
|
from the provider result
|
|
187
|
-
* :attr:`~eodag.config.PluginConfig.
|
|
188
|
-
(``dict[str, str]``): mapping for
|
|
189
|
-
|
|
190
|
-
* :attr:`~eodag.config.PluginConfig.
|
|
191
|
-
(``dict[str, str]``): mapping for
|
|
192
|
-
|
|
193
|
-
* :attr:`~eodag.config.PluginConfig.
|
|
194
|
-
data for a single collection; used if
|
|
195
|
-
:attr:`~eodag.config.PluginConfig.
|
|
196
|
-
* :attr:`~eodag.config.PluginConfig.
|
|
197
|
-
to be added to the :attr:`~eodag.config.PluginConfig.
|
|
187
|
+
* :attr:`~eodag.config.PluginConfig.DiscoverCollections.generic_collection_parsable_properties`
|
|
188
|
+
(``dict[str, str]``): mapping for collection properties which can be parsed from the result and are not
|
|
189
|
+
collection metadata
|
|
190
|
+
* :attr:`~eodag.config.PluginConfig.DiscoverCollections.generic_collection_unparsable_properties`
|
|
191
|
+
(``dict[str, str]``): mapping for collection properties which cannot be parsed from the result and are not
|
|
192
|
+
collection metadata
|
|
193
|
+
* :attr:`~eodag.config.PluginConfig.DiscoverCollections.single_collection_fetch_url` (``str``): url to fetch
|
|
194
|
+
data for a single collection; used if collection metadata is not available from the endpoint given in
|
|
195
|
+
:attr:`~eodag.config.PluginConfig.DiscoverCollections.fetch_url`
|
|
196
|
+
* :attr:`~eodag.config.PluginConfig.DiscoverCollections.single_collection_fetch_qs` (``str``): query string
|
|
197
|
+
to be added to the :attr:`~eodag.config.PluginConfig.DiscoverCollections.fetch_url` to filter for a
|
|
198
198
|
collection
|
|
199
|
-
* :attr:`~eodag.config.PluginConfig.
|
|
200
|
-
(``dict[str, str]``): mapping for
|
|
201
|
-
:attr:`~eodag.config.PluginConfig.
|
|
199
|
+
* :attr:`~eodag.config.PluginConfig.DiscoverCollections.single_collection_parsable_metadata`
|
|
200
|
+
(``dict[str, str]``): mapping for collection metadata returned by the endpoint given in
|
|
201
|
+
:attr:`~eodag.config.PluginConfig.DiscoverCollections.single_collection_fetch_url`.
|
|
202
202
|
|
|
203
203
|
* :attr:`~eodag.config.PluginConfig.sort` (:class:`~eodag.config.PluginConfig.Sort`): configuration for sorting
|
|
204
204
|
the results. It contains the keys:
|
|
@@ -232,15 +232,15 @@ class QueryStringSearch(Search):
|
|
|
232
232
|
specification of Python string formatting, with a special behaviour added to it. For example,
|
|
233
233
|
an entry in the metadata mapping of this kind::
|
|
234
234
|
|
|
235
|
-
|
|
236
|
-
- 'f=acquisition.endViewingDate:lte:{
|
|
235
|
+
end_datetime:
|
|
236
|
+
- 'f=acquisition.endViewingDate:lte:{end_datetime#timestamp}'
|
|
237
237
|
- '$.properties.acquisition.endViewingDate'
|
|
238
238
|
|
|
239
239
|
means that the search url will have a query string parameter named ``f`` with a value of
|
|
240
240
|
``acquisition.endViewingDate:lte:1543922280.0`` if the search was done with the value
|
|
241
|
-
of ``
|
|
242
|
-
``{
|
|
243
|
-
of ``
|
|
241
|
+
of ``end_datetime`` being ``2018-12-04T12:18:00``. What happened is that
|
|
242
|
+
``{end_datetime#timestamp}`` was replaced with the timestamp of the value
|
|
243
|
+
of ``end_datetime``. This example shows all there is to know about the
|
|
244
244
|
semantics of the query string formatting introduced by this plugin: any eodag search parameter
|
|
245
245
|
can be referenced in the query string with an additional optional conversion function that
|
|
246
246
|
is separated from it by a ``#`` (see :func:`~eodag.api.product.metadata_mapping.format_metadata` for further
|
|
@@ -270,16 +270,16 @@ class QueryStringSearch(Search):
|
|
|
270
270
|
provider queryables endpoint; It has the following keys:
|
|
271
271
|
|
|
272
272
|
* :attr:`~eodag.config.PluginConfig.DiscoverQueryables.fetch_url` (``str``): url to fetch the queryables valid
|
|
273
|
-
for all
|
|
274
|
-
* :attr:`~eodag.config.PluginConfig.DiscoverQueryables.
|
|
275
|
-
queryables for a specific
|
|
273
|
+
for all collections
|
|
274
|
+
* :attr:`~eodag.config.PluginConfig.DiscoverQueryables.collection_fetch_url` (``str``): url to fetch the
|
|
275
|
+
queryables for a specific collection
|
|
276
276
|
* :attr:`~eodag.config.PluginConfig.DiscoverQueryables.result_type` (``str``): type of the result (currently
|
|
277
277
|
only ``json`` is used)
|
|
278
278
|
* :attr:`~eodag.config.PluginConfig.DiscoverQueryables.results_entry` (``str``): json path to retrieve the
|
|
279
279
|
queryables from the provider result
|
|
280
280
|
|
|
281
281
|
* :attr:`~eodag.config.PluginConfig.constraints_file_url` (``str``): url to fetch the constraints for a specific
|
|
282
|
-
|
|
282
|
+
collection, can be an http url or a path to a file; the constraints are used to build queryables
|
|
283
283
|
* :attr:`~eodag.config.PluginConfig.constraints_entry` (``str``): key in the json result where the constraints
|
|
284
284
|
can be found; if not given, it is assumed that the constraints are on top level of the result, i.e.
|
|
285
285
|
the result is an array of constraints
|
|
@@ -332,45 +332,43 @@ class QueryStringSearch(Search):
|
|
|
332
332
|
self.config.pagination.get("next_page_merge_key_path")
|
|
333
333
|
)
|
|
334
334
|
|
|
335
|
-
# parse jsonpath on init:
|
|
335
|
+
# parse jsonpath on init: collections discovery
|
|
336
336
|
if (
|
|
337
|
-
getattr(self.config, "
|
|
338
|
-
and getattr(self.config, "
|
|
337
|
+
getattr(self.config, "discover_collections", {}).get("results_entry")
|
|
338
|
+
and getattr(self.config, "discover_collections", {}).get("result_type")
|
|
339
339
|
== "json"
|
|
340
340
|
):
|
|
341
|
-
self.config.
|
|
342
|
-
self.config.
|
|
341
|
+
self.config.discover_collections["results_entry"] = string_to_jsonpath(
|
|
342
|
+
self.config.discover_collections["results_entry"], force=True
|
|
343
343
|
)
|
|
344
|
-
self.config.
|
|
345
|
-
"
|
|
344
|
+
self.config.discover_collections[
|
|
345
|
+
"generic_collection_id"
|
|
346
346
|
] = mtd_cfg_as_conversion_and_querypath(
|
|
347
|
-
{"foo": self.config.
|
|
347
|
+
{"foo": self.config.discover_collections["generic_collection_id"]}
|
|
348
348
|
)[
|
|
349
349
|
"foo"
|
|
350
350
|
]
|
|
351
|
-
self.config.
|
|
352
|
-
"
|
|
351
|
+
self.config.discover_collections[
|
|
352
|
+
"generic_collection_parsable_properties"
|
|
353
353
|
] = mtd_cfg_as_conversion_and_querypath(
|
|
354
|
-
self.config.
|
|
355
|
-
"
|
|
354
|
+
self.config.discover_collections[
|
|
355
|
+
"generic_collection_parsable_properties"
|
|
356
356
|
]
|
|
357
357
|
)
|
|
358
|
-
self.config.
|
|
359
|
-
"
|
|
358
|
+
self.config.discover_collections[
|
|
359
|
+
"generic_collection_parsable_metadata"
|
|
360
360
|
] = mtd_cfg_as_conversion_and_querypath(
|
|
361
|
-
self.config.
|
|
362
|
-
"generic_product_type_parsable_metadata"
|
|
363
|
-
]
|
|
361
|
+
self.config.discover_collections["generic_collection_parsable_metadata"]
|
|
364
362
|
)
|
|
365
363
|
if (
|
|
366
|
-
"
|
|
367
|
-
in self.config.
|
|
364
|
+
"single_collection_parsable_metadata"
|
|
365
|
+
in self.config.discover_collections
|
|
368
366
|
):
|
|
369
|
-
self.config.
|
|
370
|
-
"
|
|
367
|
+
self.config.discover_collections[
|
|
368
|
+
"single_collection_parsable_metadata"
|
|
371
369
|
] = mtd_cfg_as_conversion_and_querypath(
|
|
372
|
-
self.config.
|
|
373
|
-
"
|
|
370
|
+
self.config.discover_collections[
|
|
371
|
+
"single_collection_parsable_metadata"
|
|
374
372
|
]
|
|
375
373
|
)
|
|
376
374
|
|
|
@@ -384,62 +382,60 @@ class QueryStringSearch(Search):
|
|
|
384
382
|
self.config.discover_queryables["results_entry"], force=True
|
|
385
383
|
)
|
|
386
384
|
|
|
387
|
-
# parse jsonpath on init:
|
|
388
|
-
for
|
|
385
|
+
# parse jsonpath on init: collection specific metadata-mapping
|
|
386
|
+
for collection in self.config.products.keys():
|
|
389
387
|
|
|
390
|
-
|
|
391
|
-
#
|
|
388
|
+
collection_metadata_mapping = {}
|
|
389
|
+
# collection specific metadata-mapping
|
|
392
390
|
if any(
|
|
393
|
-
mm in self.config.products[
|
|
391
|
+
mm in self.config.products[collection].keys()
|
|
394
392
|
for mm in ("metadata_mapping", "metadata_mapping_from_product")
|
|
395
393
|
):
|
|
396
|
-
# Complete and ready to use
|
|
397
|
-
|
|
394
|
+
# Complete and ready to use collection specific metadata-mapping
|
|
395
|
+
collection_metadata_mapping = deepcopy(self.config.metadata_mapping)
|
|
398
396
|
|
|
399
397
|
# metadata_mapping from another product
|
|
400
|
-
if other_product_for_mapping := self.config.products[
|
|
398
|
+
if other_product_for_mapping := self.config.products[collection].get(
|
|
401
399
|
"metadata_mapping_from_product"
|
|
402
400
|
):
|
|
403
|
-
|
|
401
|
+
other_collection_def_params = self.get_collection_def_params(
|
|
404
402
|
other_product_for_mapping,
|
|
405
403
|
)
|
|
406
404
|
# parse mapping to apply
|
|
407
|
-
if
|
|
405
|
+
if other_collection_mtd_mapping := other_collection_def_params.get(
|
|
408
406
|
"metadata_mapping", {}
|
|
409
407
|
):
|
|
410
|
-
|
|
411
|
-
|
|
412
|
-
other_product_type_def_params.get("metadata_mapping", {})
|
|
413
|
-
)
|
|
408
|
+
other_collection_mtd_mapping = mtd_cfg_as_conversion_and_querypath(
|
|
409
|
+
other_collection_def_params.get("metadata_mapping", {})
|
|
414
410
|
)
|
|
415
411
|
else:
|
|
416
|
-
msg = f"Cannot reuse empty metadata_mapping from {other_product_for_mapping} for {
|
|
412
|
+
msg = f"Cannot reuse empty metadata_mapping from {other_product_for_mapping} for {collection}"
|
|
417
413
|
raise MisconfiguredError(msg)
|
|
418
414
|
# update mapping
|
|
419
|
-
for metadata, mapping in
|
|
420
|
-
|
|
421
|
-
|
|
415
|
+
for metadata, mapping in other_collection_mtd_mapping.items():
|
|
416
|
+
collection_metadata_mapping.pop(metadata, None)
|
|
417
|
+
collection_metadata_mapping[metadata] = mapping
|
|
422
418
|
|
|
423
419
|
# metadata_mapping from current product
|
|
424
|
-
if "metadata_mapping" in self.config.products[
|
|
420
|
+
if "metadata_mapping" in self.config.products[collection].keys():
|
|
425
421
|
# parse mapping to apply
|
|
426
|
-
self.config.products[
|
|
422
|
+
self.config.products[collection][
|
|
427
423
|
"metadata_mapping"
|
|
428
424
|
] = mtd_cfg_as_conversion_and_querypath(
|
|
429
|
-
self.config.products[
|
|
425
|
+
self.config.products[collection]["metadata_mapping"]
|
|
430
426
|
)
|
|
431
427
|
|
|
432
428
|
# from current product, updated mapping at the end
|
|
433
|
-
for metadata, mapping in self.config.products[
|
|
429
|
+
for metadata, mapping in self.config.products[collection][
|
|
434
430
|
"metadata_mapping"
|
|
435
431
|
].items():
|
|
436
|
-
|
|
437
|
-
|
|
432
|
+
collection_metadata_mapping.pop(metadata, None)
|
|
433
|
+
collection_metadata_mapping[metadata] = mapping
|
|
438
434
|
|
|
439
|
-
if
|
|
440
|
-
self.config.products[
|
|
435
|
+
if collection_metadata_mapping:
|
|
436
|
+
self.config.products[collection][
|
|
441
437
|
"metadata_mapping"
|
|
442
|
-
] =
|
|
438
|
+
] = collection_metadata_mapping
|
|
443
439
|
|
|
444
440
|
def clear(self) -> None:
|
|
445
441
|
"""Clear search context"""
|
|
@@ -451,32 +447,32 @@ class QueryStringSearch(Search):
|
|
|
451
447
|
self.next_page_query_obj = None
|
|
452
448
|
self.next_page_merge = None
|
|
453
449
|
|
|
454
|
-
def
|
|
455
|
-
"""Fetch
|
|
450
|
+
def discover_collections(self, **kwargs: Any) -> Optional[dict[str, Any]]:
|
|
451
|
+
"""Fetch collections list from provider using `discover_collections` conf
|
|
456
452
|
|
|
457
|
-
:returns: configuration dict containing fetched
|
|
453
|
+
:returns: configuration dict containing fetched collections information
|
|
458
454
|
"""
|
|
459
|
-
unpaginated_fetch_url = self.config.
|
|
455
|
+
unpaginated_fetch_url = self.config.discover_collections.get("fetch_url")
|
|
460
456
|
if not unpaginated_fetch_url:
|
|
461
457
|
return None
|
|
462
458
|
|
|
463
|
-
#
|
|
464
|
-
next_page_url_tpl = self.config.
|
|
465
|
-
page = self.config.
|
|
459
|
+
# collections pagination
|
|
460
|
+
next_page_url_tpl = self.config.discover_collections.get("next_page_url_tpl")
|
|
461
|
+
page = self.config.discover_collections.get("start_page", 1)
|
|
466
462
|
|
|
467
463
|
if not next_page_url_tpl:
|
|
468
464
|
# no pagination
|
|
469
|
-
return self.
|
|
465
|
+
return self.discover_collections_per_page(**kwargs)
|
|
470
466
|
|
|
471
467
|
conf_update_dict: dict[str, Any] = {
|
|
472
468
|
"providers_config": {},
|
|
473
|
-
"
|
|
469
|
+
"collections_config": {},
|
|
474
470
|
}
|
|
475
471
|
|
|
476
472
|
while True:
|
|
477
473
|
fetch_url = next_page_url_tpl.format(url=unpaginated_fetch_url, page=page)
|
|
478
474
|
|
|
479
|
-
conf_update_dict_per_page = self.
|
|
475
|
+
conf_update_dict_per_page = self.discover_collections_per_page(
|
|
480
476
|
fetch_url=fetch_url, **kwargs
|
|
481
477
|
)
|
|
482
478
|
|
|
@@ -491,29 +487,27 @@ class QueryStringSearch(Search):
|
|
|
491
487
|
conf_update_dict["providers_config"].update(
|
|
492
488
|
conf_update_dict_per_page["providers_config"]
|
|
493
489
|
)
|
|
494
|
-
conf_update_dict["
|
|
495
|
-
conf_update_dict_per_page["
|
|
490
|
+
conf_update_dict["collections_config"].update(
|
|
491
|
+
conf_update_dict_per_page["collections_config"]
|
|
496
492
|
)
|
|
497
493
|
|
|
498
494
|
page += 1
|
|
499
495
|
|
|
500
496
|
return conf_update_dict
|
|
501
497
|
|
|
502
|
-
def
|
|
503
|
-
|
|
504
|
-
) -> Optional[dict[str, Any]]:
|
|
505
|
-
"""Fetch product types list from provider using `discover_product_types` conf
|
|
498
|
+
def discover_collections_per_page(self, **kwargs: Any) -> Optional[dict[str, Any]]:
|
|
499
|
+
"""Fetch collections list from provider using `discover_collections` conf
|
|
506
500
|
using paginated ``kwargs["fetch_url"]``
|
|
507
501
|
|
|
508
|
-
:returns: configuration dict containing fetched
|
|
502
|
+
:returns: configuration dict containing fetched collections information
|
|
509
503
|
"""
|
|
510
504
|
try:
|
|
511
505
|
prep = PreparedSearch()
|
|
512
506
|
|
|
513
|
-
# url from
|
|
507
|
+
# url from discover_collections() or conf
|
|
514
508
|
fetch_url: Optional[str] = kwargs.get("fetch_url")
|
|
515
509
|
if fetch_url is None:
|
|
516
|
-
if fetch_url := self.config.
|
|
510
|
+
if fetch_url := self.config.discover_collections.get("fetch_url"):
|
|
517
511
|
fetch_url = fetch_url.format(**self.config.__dict__)
|
|
518
512
|
else:
|
|
519
513
|
return None
|
|
@@ -523,13 +517,15 @@ class QueryStringSearch(Search):
|
|
|
523
517
|
if "auth" in kwargs:
|
|
524
518
|
prep.auth = kwargs.pop("auth")
|
|
525
519
|
|
|
526
|
-
# try updating fetch_url qs using
|
|
520
|
+
# try updating fetch_url qs using collection
|
|
527
521
|
fetch_qs_dict = {}
|
|
528
|
-
if "single_collection_fetch_qs" in self.config.
|
|
522
|
+
if "single_collection_fetch_qs" in self.config.discover_collections:
|
|
529
523
|
try:
|
|
530
|
-
fetch_qs =
|
|
531
|
-
|
|
532
|
-
|
|
524
|
+
fetch_qs = format_string(
|
|
525
|
+
None,
|
|
526
|
+
self.config.discover_collections["single_collection_fetch_qs"],
|
|
527
|
+
**kwargs,
|
|
528
|
+
)
|
|
533
529
|
fetch_qs_dict = dict(parse_qsl(fetch_qs))
|
|
534
530
|
except KeyError:
|
|
535
531
|
pass
|
|
@@ -542,14 +538,14 @@ class QueryStringSearch(Search):
|
|
|
542
538
|
url_parse = url_parse._replace(query=url_new_query)
|
|
543
539
|
prep.url = urlunparse(url_parse)
|
|
544
540
|
|
|
545
|
-
prep.info_message = "Fetching
|
|
541
|
+
prep.info_message = "Fetching collections: {}".format(prep.url)
|
|
546
542
|
prep.exception_message = (
|
|
547
|
-
"Skipping error while fetching
|
|
543
|
+
"Skipping error while fetching collections for {} {} instance:"
|
|
548
544
|
).format(self.provider, self.__class__.__name__)
|
|
549
545
|
|
|
550
546
|
# Query using appropriate method
|
|
551
|
-
fetch_method = self.config.
|
|
552
|
-
fetch_body = self.config.
|
|
547
|
+
fetch_method = self.config.discover_collections.get("fetch_method", "GET")
|
|
548
|
+
fetch_body = self.config.discover_collections.get("fetch_body", {})
|
|
553
549
|
if fetch_method == "POST" and isinstance(self, PostJsonSearch):
|
|
554
550
|
prep.query_params = fetch_body
|
|
555
551
|
response = self._request(prep)
|
|
@@ -561,15 +557,15 @@ class QueryStringSearch(Search):
|
|
|
561
557
|
try:
|
|
562
558
|
conf_update_dict: dict[str, Any] = {
|
|
563
559
|
"providers_config": {},
|
|
564
|
-
"
|
|
560
|
+
"collections_config": {},
|
|
565
561
|
}
|
|
566
|
-
if self.config.
|
|
562
|
+
if self.config.discover_collections["result_type"] == "json":
|
|
567
563
|
resp_as_json = response.json()
|
|
568
564
|
# extract results from response json
|
|
569
|
-
results_entry = self.config.
|
|
565
|
+
results_entry = self.config.discover_collections["results_entry"]
|
|
570
566
|
if not isinstance(results_entry, JSONPath):
|
|
571
567
|
logger.warning(
|
|
572
|
-
f"Could not parse {self.provider}
|
|
568
|
+
f"Could not parse {self.provider} discover_collections.results_entry"
|
|
573
569
|
f" as JSONPath: {results_entry}"
|
|
574
570
|
)
|
|
575
571
|
return None
|
|
@@ -577,99 +573,99 @@ class QueryStringSearch(Search):
|
|
|
577
573
|
if result and isinstance(result[0], list):
|
|
578
574
|
result = result[0]
|
|
579
575
|
|
|
580
|
-
def
|
|
581
|
-
|
|
576
|
+
def conf_update_from_collection_result(
|
|
577
|
+
collection_result: dict[str, Any],
|
|
582
578
|
) -> None:
|
|
583
|
-
"""Update ``conf_update_dict`` using given
|
|
579
|
+
"""Update ``conf_update_dict`` using given collection json response"""
|
|
584
580
|
# providers_config extraction
|
|
585
581
|
extracted_mapping = properties_from_json(
|
|
586
|
-
|
|
582
|
+
collection_result,
|
|
587
583
|
dict(
|
|
588
|
-
self.config.
|
|
589
|
-
"
|
|
584
|
+
self.config.discover_collections[
|
|
585
|
+
"generic_collection_parsable_properties"
|
|
590
586
|
],
|
|
591
587
|
**{
|
|
592
|
-
"
|
|
593
|
-
"
|
|
588
|
+
"generic_collection_id": self.config.discover_collections[
|
|
589
|
+
"generic_collection_id"
|
|
594
590
|
]
|
|
595
591
|
},
|
|
596
592
|
),
|
|
597
593
|
)
|
|
598
|
-
|
|
599
|
-
"
|
|
594
|
+
generic_collection_id = extracted_mapping.pop(
|
|
595
|
+
"generic_collection_id"
|
|
600
596
|
)
|
|
601
597
|
conf_update_dict["providers_config"][
|
|
602
|
-
|
|
598
|
+
generic_collection_id
|
|
603
599
|
] = dict(
|
|
604
600
|
extracted_mapping,
|
|
605
|
-
**self.config.
|
|
606
|
-
"
|
|
601
|
+
**self.config.discover_collections.get(
|
|
602
|
+
"generic_collection_unparsable_properties", {}
|
|
607
603
|
),
|
|
608
604
|
)
|
|
609
|
-
#
|
|
610
|
-
conf_update_dict["
|
|
611
|
-
|
|
605
|
+
# collections_config extraction
|
|
606
|
+
conf_update_dict["collections_config"][
|
|
607
|
+
generic_collection_id
|
|
612
608
|
] = properties_from_json(
|
|
613
|
-
|
|
614
|
-
self.config.
|
|
615
|
-
"
|
|
609
|
+
collection_result,
|
|
610
|
+
self.config.discover_collections[
|
|
611
|
+
"generic_collection_parsable_metadata"
|
|
616
612
|
],
|
|
617
613
|
)
|
|
618
614
|
|
|
619
615
|
if (
|
|
620
|
-
"
|
|
621
|
-
in self.config.
|
|
616
|
+
"single_collection_parsable_metadata"
|
|
617
|
+
in self.config.discover_collections
|
|
622
618
|
):
|
|
623
|
-
collection_data = self.
|
|
624
|
-
|
|
619
|
+
collection_data = self._get_collection_metadata_from_single_collection_endpoint(
|
|
620
|
+
generic_collection_id
|
|
625
621
|
)
|
|
626
|
-
conf_update_dict["
|
|
627
|
-
|
|
622
|
+
conf_update_dict["collections_config"][
|
|
623
|
+
generic_collection_id
|
|
628
624
|
].update(collection_data)
|
|
629
625
|
|
|
630
|
-
# update
|
|
626
|
+
# update collection id if needed
|
|
631
627
|
if collection_data_id := collection_data.get("ID"):
|
|
632
|
-
if
|
|
628
|
+
if generic_collection_id != collection_data_id:
|
|
633
629
|
logger.debug(
|
|
634
|
-
"Rename %s
|
|
635
|
-
|
|
630
|
+
"Rename %s collection to %s",
|
|
631
|
+
generic_collection_id,
|
|
636
632
|
collection_data_id,
|
|
637
633
|
)
|
|
638
634
|
conf_update_dict["providers_config"][
|
|
639
635
|
collection_data_id
|
|
640
636
|
] = conf_update_dict["providers_config"].pop(
|
|
641
|
-
|
|
637
|
+
generic_collection_id
|
|
642
638
|
)
|
|
643
|
-
conf_update_dict["
|
|
639
|
+
conf_update_dict["collections_config"][
|
|
644
640
|
collection_data_id
|
|
645
|
-
] = conf_update_dict["
|
|
646
|
-
|
|
641
|
+
] = conf_update_dict["collections_config"].pop(
|
|
642
|
+
generic_collection_id
|
|
647
643
|
)
|
|
648
|
-
|
|
644
|
+
generic_collection_id = collection_data_id
|
|
649
645
|
|
|
650
646
|
# update keywords
|
|
651
647
|
keywords_fields = [
|
|
652
|
-
"
|
|
648
|
+
"instruments",
|
|
649
|
+
"constellation",
|
|
653
650
|
"platform",
|
|
654
|
-
"
|
|
655
|
-
"processingLevel",
|
|
651
|
+
"processing:level",
|
|
656
652
|
"keywords",
|
|
657
653
|
]
|
|
658
654
|
keywords_values_str = ",".join(
|
|
659
|
-
[
|
|
655
|
+
[generic_collection_id]
|
|
660
656
|
+ [
|
|
661
657
|
str(
|
|
662
|
-
conf_update_dict["
|
|
663
|
-
|
|
658
|
+
conf_update_dict["collections_config"][
|
|
659
|
+
generic_collection_id
|
|
664
660
|
][kf]
|
|
665
661
|
)
|
|
666
662
|
for kf in keywords_fields
|
|
667
663
|
if kf
|
|
668
|
-
in conf_update_dict["
|
|
669
|
-
|
|
664
|
+
in conf_update_dict["collections_config"][
|
|
665
|
+
generic_collection_id
|
|
670
666
|
]
|
|
671
|
-
and conf_update_dict["
|
|
672
|
-
|
|
667
|
+
and conf_update_dict["collections_config"][
|
|
668
|
+
generic_collection_id
|
|
673
669
|
][kf]
|
|
674
670
|
!= NOT_AVAILABLE
|
|
675
671
|
]
|
|
@@ -688,66 +684,66 @@ class QueryStringSearch(Search):
|
|
|
688
684
|
keywords_values_str = ",".join(
|
|
689
685
|
sorted(set(keywords_values_str.split(",")))
|
|
690
686
|
)
|
|
691
|
-
conf_update_dict["
|
|
692
|
-
|
|
693
|
-
]
|
|
687
|
+
conf_update_dict["collections_config"][generic_collection_id][
|
|
688
|
+
"keywords"
|
|
689
|
+
] = keywords_values_str
|
|
694
690
|
|
|
695
691
|
# runs concurrent requests and aggregate results in conf_update_dict
|
|
696
|
-
max_connections = self.config.
|
|
692
|
+
max_connections = self.config.discover_collections.get(
|
|
697
693
|
"max_connections"
|
|
698
694
|
)
|
|
699
695
|
with concurrent.futures.ThreadPoolExecutor(
|
|
700
696
|
max_workers=max_connections
|
|
701
697
|
) as executor:
|
|
702
698
|
futures = (
|
|
703
|
-
executor.submit(
|
|
699
|
+
executor.submit(conf_update_from_collection_result, r)
|
|
704
700
|
for r in result
|
|
705
701
|
)
|
|
706
702
|
[f.result() for f in concurrent.futures.as_completed(futures)]
|
|
707
703
|
|
|
708
704
|
except KeyError as e:
|
|
709
705
|
logger.warning(
|
|
710
|
-
"Incomplete %s
|
|
706
|
+
"Incomplete %s discover_collections configuration: %s",
|
|
711
707
|
self.provider,
|
|
712
708
|
e,
|
|
713
709
|
)
|
|
714
710
|
return None
|
|
715
711
|
except requests.RequestException as e:
|
|
716
712
|
logger.debug(
|
|
717
|
-
"Could not parse discovered
|
|
713
|
+
"Could not parse discovered collections response from "
|
|
718
714
|
f"{self.provider}, {type(e).__name__}: {e.args}"
|
|
719
715
|
)
|
|
720
716
|
return None
|
|
721
|
-
conf_update_dict["
|
|
722
|
-
conf_update_dict["
|
|
717
|
+
conf_update_dict["collections_config"] = dict_items_recursive_apply(
|
|
718
|
+
conf_update_dict["collections_config"],
|
|
723
719
|
lambda k, v: v if v != NOT_AVAILABLE else None,
|
|
724
720
|
)
|
|
725
721
|
return conf_update_dict
|
|
726
722
|
|
|
727
|
-
def
|
|
728
|
-
self,
|
|
723
|
+
def _get_collection_metadata_from_single_collection_endpoint(
|
|
724
|
+
self, collection: str
|
|
729
725
|
) -> dict[str, Any]:
|
|
730
726
|
"""
|
|
731
|
-
retrieves additional
|
|
732
|
-
:param
|
|
733
|
-
:return:
|
|
727
|
+
retrieves additional collection information from an endpoint returning data for a single collection
|
|
728
|
+
:param collection: collection
|
|
729
|
+
:return: collections and their metadata
|
|
734
730
|
"""
|
|
735
|
-
single_collection_url = self.config.
|
|
731
|
+
single_collection_url = self.config.discover_collections[
|
|
736
732
|
"single_collection_fetch_url"
|
|
737
|
-
].format(
|
|
733
|
+
].format(_collection=collection)
|
|
738
734
|
resp = QueryStringSearch._request(
|
|
739
735
|
self,
|
|
740
736
|
PreparedSearch(
|
|
741
737
|
url=single_collection_url,
|
|
742
|
-
info_message=f"Fetching data for
|
|
743
|
-
exception_message="Skipping error while fetching
|
|
738
|
+
info_message=f"Fetching data for collection: {collection}",
|
|
739
|
+
exception_message="Skipping error while fetching collections for "
|
|
744
740
|
"{} {} instance:".format(self.provider, self.__class__.__name__),
|
|
745
741
|
),
|
|
746
742
|
)
|
|
747
743
|
product_data = resp.json()
|
|
748
744
|
return properties_from_json(
|
|
749
745
|
product_data,
|
|
750
|
-
self.config.
|
|
746
|
+
self.config.discover_collections["single_collection_parsable_metadata"],
|
|
751
747
|
)
|
|
752
748
|
|
|
753
749
|
def query(
|
|
@@ -760,10 +756,10 @@ class QueryStringSearch(Search):
|
|
|
760
756
|
:param prep: Object collecting needed information for search.
|
|
761
757
|
"""
|
|
762
758
|
count = prep.count
|
|
763
|
-
|
|
764
|
-
if
|
|
759
|
+
collection = cast(str, kwargs.get("collection", prep.collection))
|
|
760
|
+
if collection == GENERIC_COLLECTION:
|
|
765
761
|
logger.warning(
|
|
766
|
-
"
|
|
762
|
+
"GENERIC_COLLECTION is not a real collection and should only be used internally as a template"
|
|
767
763
|
)
|
|
768
764
|
return ([], 0) if prep.count else ([], None)
|
|
769
765
|
|
|
@@ -772,41 +768,41 @@ class QueryStringSearch(Search):
|
|
|
772
768
|
("", {}) if sort_by_arg is None else self.build_sort_by(sort_by_arg)
|
|
773
769
|
)
|
|
774
770
|
|
|
775
|
-
|
|
771
|
+
provider_collection = self.map_collection(collection)
|
|
776
772
|
keywords = {k: v for k, v in kwargs.items() if k != "auth" and v is not None}
|
|
777
|
-
keywords["
|
|
778
|
-
|
|
779
|
-
if (
|
|
780
|
-
else
|
|
773
|
+
keywords["collection"] = (
|
|
774
|
+
provider_collection
|
|
775
|
+
if (provider_collection and provider_collection != GENERIC_COLLECTION)
|
|
776
|
+
else collection
|
|
781
777
|
)
|
|
782
778
|
|
|
783
|
-
# provider
|
|
784
|
-
prep.
|
|
785
|
-
self.
|
|
786
|
-
if
|
|
779
|
+
# provider collection specific conf
|
|
780
|
+
prep.collection_def_params = (
|
|
781
|
+
self.get_collection_def_params(collection, format_variables=kwargs)
|
|
782
|
+
if collection is not None
|
|
787
783
|
else {}
|
|
788
784
|
)
|
|
789
785
|
|
|
790
|
-
# if
|
|
791
|
-
if prep.
|
|
792
|
-
keywords.pop("
|
|
786
|
+
# if collection_def_params is set, remove collection as it may conflict with this conf
|
|
787
|
+
if prep.collection_def_params:
|
|
788
|
+
keywords.pop("collection", None)
|
|
793
789
|
|
|
794
790
|
if self.config.metadata_mapping:
|
|
795
|
-
|
|
791
|
+
collection_metadata_mapping = dict(
|
|
796
792
|
self.config.metadata_mapping,
|
|
797
|
-
**prep.
|
|
793
|
+
**prep.collection_def_params.get("metadata_mapping", {}),
|
|
798
794
|
)
|
|
799
795
|
keywords.update(
|
|
800
796
|
{
|
|
801
797
|
k: v
|
|
802
|
-
for k, v in prep.
|
|
798
|
+
for k, v in prep.collection_def_params.items()
|
|
803
799
|
if k not in keywords.keys()
|
|
804
|
-
and k in
|
|
805
|
-
and isinstance(
|
|
800
|
+
and k in collection_metadata_mapping.keys()
|
|
801
|
+
and isinstance(collection_metadata_mapping[k], list)
|
|
806
802
|
}
|
|
807
803
|
)
|
|
808
804
|
|
|
809
|
-
qp, qs = self.build_query_string(
|
|
805
|
+
qp, qs = self.build_query_string(collection, keywords)
|
|
810
806
|
|
|
811
807
|
prep.query_params = qp
|
|
812
808
|
prep.query_string = qs
|
|
@@ -825,28 +821,19 @@ class QueryStringSearch(Search):
|
|
|
825
821
|
|
|
826
822
|
raw_search_result = RawSearchResult(provider_results)
|
|
827
823
|
raw_search_result.query_params = prep.query_params
|
|
828
|
-
raw_search_result.
|
|
824
|
+
raw_search_result.collection_def_params = prep.collection_def_params
|
|
829
825
|
|
|
830
826
|
eo_products = self.normalize_results(raw_search_result, **kwargs)
|
|
831
827
|
return eo_products, total_items
|
|
832
828
|
|
|
833
|
-
@_deprecated(
|
|
834
|
-
reason="Simply run `self.config.metadata_mapping.update(metadata_mapping)` instead",
|
|
835
|
-
version="2.10.0",
|
|
836
|
-
)
|
|
837
|
-
def update_metadata_mapping(self, metadata_mapping: dict[str, Any]) -> None:
|
|
838
|
-
"""Update plugin metadata_mapping with input metadata_mapping configuration"""
|
|
839
|
-
if self.config.metadata_mapping:
|
|
840
|
-
self.config.metadata_mapping.update(metadata_mapping)
|
|
841
|
-
|
|
842
829
|
def build_query_string(
|
|
843
|
-
self,
|
|
830
|
+
self, collection: str, query_dict: dict[str, Any]
|
|
844
831
|
) -> tuple[dict[str, Any], str]:
|
|
845
832
|
"""Build The query string using the search parameters"""
|
|
846
833
|
logger.debug("Building the query string that will be used for search")
|
|
847
|
-
error_context = f"
|
|
834
|
+
error_context = f"Collection: {collection} / provider : {self.provider}"
|
|
848
835
|
query_params = format_query_params(
|
|
849
|
-
|
|
836
|
+
collection, self.config, query_dict, error_context
|
|
850
837
|
)
|
|
851
838
|
|
|
852
839
|
# Build the final query string, in one go without quoting it
|
|
@@ -885,19 +872,21 @@ class QueryStringSearch(Search):
|
|
|
885
872
|
prep.need_count = True
|
|
886
873
|
prep.total_items_nb = None
|
|
887
874
|
|
|
888
|
-
for
|
|
875
|
+
for provider_collection in self.get_provider_collections(prep, **kwargs) or (
|
|
876
|
+
None,
|
|
877
|
+
):
|
|
889
878
|
# skip empty collection if one is required in api_endpoint
|
|
890
|
-
if "{
|
|
879
|
+
if "{_collection}" in self.config.api_endpoint and not provider_collection:
|
|
891
880
|
continue
|
|
892
881
|
search_endpoint = self.config.api_endpoint.rstrip("/").format(
|
|
893
|
-
|
|
882
|
+
_collection=provider_collection
|
|
894
883
|
)
|
|
895
884
|
if page is not None and items_per_page is not None:
|
|
896
885
|
page = page - 1 + self.config.pagination.get("start_page", 1)
|
|
897
886
|
if count:
|
|
898
887
|
count_endpoint = self.config.pagination.get(
|
|
899
888
|
"count_endpoint", ""
|
|
900
|
-
).format(
|
|
889
|
+
).format(_collection=provider_collection)
|
|
901
890
|
if count_endpoint:
|
|
902
891
|
count_url = "{}?{}".format(count_endpoint, prep.query_string)
|
|
903
892
|
_total_results = (
|
|
@@ -1110,19 +1099,16 @@ class QueryStringSearch(Search):
|
|
|
1110
1099
|
products: list[EOProduct] = []
|
|
1111
1100
|
asset_key_from_href = getattr(self.config, "asset_key_from_href", True)
|
|
1112
1101
|
for result in results:
|
|
1113
|
-
|
|
1114
|
-
|
|
1115
|
-
|
|
1116
|
-
|
|
1117
|
-
self.get_metadata_mapping(kwargs.get("productType")),
|
|
1118
|
-
discovery_config=getattr(self.config, "discover_metadata", {}),
|
|
1119
|
-
),
|
|
1120
|
-
**kwargs,
|
|
1121
|
-
)
|
|
1122
|
-
# use product_type_config as default properties
|
|
1123
|
-
product.properties = dict(
|
|
1124
|
-
getattr(self.config, "product_type_config", {}), **product.properties
|
|
1102
|
+
properties = QueryStringSearch.extract_properties[self.config.result_type](
|
|
1103
|
+
result,
|
|
1104
|
+
self.get_metadata_mapping(kwargs.get("collection")),
|
|
1105
|
+
discovery_config=getattr(self.config, "discover_metadata", {}),
|
|
1125
1106
|
)
|
|
1107
|
+
# collection alias (required by opentelemetry-instrumentation-eodag)
|
|
1108
|
+
if alias := getattr(self.config, "collection_config", {}).get("alias"):
|
|
1109
|
+
properties["eodag:alias"] = alias
|
|
1110
|
+
product = EOProduct(self.provider, properties, **kwargs)
|
|
1111
|
+
|
|
1126
1112
|
additional_assets = self.get_assets_from_mapping(result)
|
|
1127
1113
|
product.assets.update(additional_assets)
|
|
1128
1114
|
# move assets from properties to product's attr, normalize keys & roles
|
|
@@ -1178,49 +1164,47 @@ class QueryStringSearch(Search):
|
|
|
1178
1164
|
total_results = int(count_results)
|
|
1179
1165
|
return total_results
|
|
1180
1166
|
|
|
1181
|
-
def
|
|
1182
|
-
|
|
1183
|
-
|
|
1184
|
-
|
|
1185
|
-
|
|
1186
|
-
|
|
1187
|
-
collection
|
|
1188
|
-
|
|
1189
|
-
not hasattr(prep, "product_type_def_params")
|
|
1190
|
-
or not prep.product_type_def_params
|
|
1167
|
+
def get_provider_collections(
|
|
1168
|
+
self, prep: PreparedSearch, **kwargs: Any
|
|
1169
|
+
) -> tuple[str, ...]:
|
|
1170
|
+
"""Get the _collection(s) / provider collection(s) to which the product belongs"""
|
|
1171
|
+
collection: Optional[str] = kwargs.get("collection")
|
|
1172
|
+
provider_collection: Optional[str] = None
|
|
1173
|
+
if collection is None and (
|
|
1174
|
+
not hasattr(prep, "collection_def_params") or not prep.collection_def_params
|
|
1191
1175
|
):
|
|
1192
1176
|
collections: set[str] = set()
|
|
1193
|
-
|
|
1194
|
-
if
|
|
1177
|
+
provider_collection = getattr(self.config, "_collection", None)
|
|
1178
|
+
if provider_collection is None:
|
|
1195
1179
|
try:
|
|
1196
|
-
for
|
|
1197
|
-
if
|
|
1198
|
-
collections.add(product_config["
|
|
1180
|
+
for collection, product_config in self.config.products.items():
|
|
1181
|
+
if collection != GENERIC_COLLECTION:
|
|
1182
|
+
collections.add(product_config["_collection"])
|
|
1199
1183
|
else:
|
|
1200
1184
|
collections.add(
|
|
1201
1185
|
format_dict_items(product_config, **kwargs).get(
|
|
1202
|
-
"
|
|
1186
|
+
"_collection", ""
|
|
1203
1187
|
)
|
|
1204
1188
|
)
|
|
1205
1189
|
except KeyError:
|
|
1206
1190
|
collections.add("")
|
|
1207
1191
|
else:
|
|
1208
|
-
collections.add(
|
|
1192
|
+
collections.add(provider_collection)
|
|
1209
1193
|
return tuple(collections)
|
|
1210
1194
|
|
|
1211
|
-
|
|
1212
|
-
if
|
|
1213
|
-
|
|
1214
|
-
getattr(prep, "
|
|
1215
|
-
or
|
|
1195
|
+
provider_collection = getattr(self.config, "_collection", None)
|
|
1196
|
+
if provider_collection is None:
|
|
1197
|
+
provider_collection = (
|
|
1198
|
+
getattr(prep, "collection_def_params", {}).get("_collection")
|
|
1199
|
+
or collection
|
|
1216
1200
|
)
|
|
1217
1201
|
|
|
1218
|
-
if
|
|
1202
|
+
if provider_collection is None:
|
|
1219
1203
|
return ()
|
|
1220
|
-
elif not isinstance(
|
|
1221
|
-
return (
|
|
1204
|
+
elif not isinstance(provider_collection, list):
|
|
1205
|
+
return (provider_collection,)
|
|
1222
1206
|
else:
|
|
1223
|
-
return tuple(
|
|
1207
|
+
return tuple(provider_collection)
|
|
1224
1208
|
|
|
1225
1209
|
def _request(
|
|
1226
1210
|
self,
|
|
@@ -1350,9 +1334,9 @@ class ODataV4Search(QueryStringSearch):
|
|
|
1350
1334
|
operations: # The operations to build
|
|
1351
1335
|
<opname>: # e.g: AND
|
|
1352
1336
|
- <op1> # e.g:
|
|
1353
|
-
# 'sensingStartDate:[{
|
|
1337
|
+
# 'sensingStartDate:[{start_datetime}Z TO *]'
|
|
1354
1338
|
- <op2> # e.g:
|
|
1355
|
-
# 'sensingStopDate:[* TO {
|
|
1339
|
+
# 'sensingStopDate:[* TO {end_datetime}Z]'
|
|
1356
1340
|
...
|
|
1357
1341
|
...
|
|
1358
1342
|
...
|
|
@@ -1484,60 +1468,60 @@ class PostJsonSearch(QueryStringSearch):
|
|
|
1484
1468
|
**kwargs: Any,
|
|
1485
1469
|
) -> tuple[list[EOProduct], Optional[int]]:
|
|
1486
1470
|
"""Perform a search on an OpenSearch-like interface"""
|
|
1487
|
-
|
|
1471
|
+
collection = kwargs.get("collection", "")
|
|
1488
1472
|
count = prep.count
|
|
1489
|
-
# remove "product_type" from search args if exists for compatibility with QueryStringSearch methods
|
|
1490
|
-
kwargs.pop("product_type", None)
|
|
1491
1473
|
sort_by_arg: Optional[SortByList] = self.get_sort_by_arg(kwargs)
|
|
1492
1474
|
_, sort_by_qp = (
|
|
1493
1475
|
("", {}) if sort_by_arg is None else self.build_sort_by(sort_by_arg)
|
|
1494
1476
|
)
|
|
1495
|
-
|
|
1477
|
+
provider_collection = self.map_collection(collection)
|
|
1496
1478
|
_dc_qs = kwargs.pop("_dc_qs", None)
|
|
1497
1479
|
if _dc_qs is not None:
|
|
1498
1480
|
qs = unquote_plus(unquote_plus(_dc_qs))
|
|
1499
1481
|
qp = geojson.loads(qs)
|
|
1500
1482
|
|
|
1501
|
-
# provider
|
|
1502
|
-
prep.
|
|
1503
|
-
|
|
1483
|
+
# provider collection specific conf
|
|
1484
|
+
prep.collection_def_params = self.get_collection_def_params(
|
|
1485
|
+
collection, format_variables=kwargs
|
|
1504
1486
|
)
|
|
1505
1487
|
else:
|
|
1506
1488
|
keywords = {
|
|
1507
|
-
k: v
|
|
1489
|
+
k: v
|
|
1490
|
+
for k, v in kwargs.items()
|
|
1491
|
+
if k not in ("auth", "collection") and v is not None
|
|
1508
1492
|
}
|
|
1509
1493
|
|
|
1510
|
-
if
|
|
1511
|
-
keywords["
|
|
1512
|
-
elif
|
|
1513
|
-
keywords["
|
|
1494
|
+
if provider_collection and provider_collection != GENERIC_COLLECTION:
|
|
1495
|
+
keywords["_collection"] = provider_collection
|
|
1496
|
+
elif collection:
|
|
1497
|
+
keywords["_collection"] = collection
|
|
1514
1498
|
|
|
1515
|
-
# provider
|
|
1516
|
-
prep.
|
|
1517
|
-
|
|
1499
|
+
# provider collection specific conf
|
|
1500
|
+
prep.collection_def_params = self.get_collection_def_params(
|
|
1501
|
+
collection, format_variables=kwargs
|
|
1518
1502
|
)
|
|
1519
1503
|
|
|
1520
|
-
# Add to the query, the queryable parameters set in the provider
|
|
1521
|
-
|
|
1504
|
+
# Add to the query, the queryable parameters set in the provider collection definition
|
|
1505
|
+
collection_metadata_mapping = {
|
|
1522
1506
|
**getattr(self.config, "metadata_mapping", {}),
|
|
1523
|
-
**prep.
|
|
1507
|
+
**prep.collection_def_params.get("metadata_mapping", {}),
|
|
1524
1508
|
}
|
|
1525
1509
|
keywords.update(
|
|
1526
1510
|
{
|
|
1527
1511
|
k: v
|
|
1528
|
-
for k, v in prep.
|
|
1512
|
+
for k, v in prep.collection_def_params.items()
|
|
1529
1513
|
if k not in keywords.keys()
|
|
1530
|
-
and k in
|
|
1531
|
-
and isinstance(
|
|
1514
|
+
and k in collection_metadata_mapping.keys()
|
|
1515
|
+
and isinstance(collection_metadata_mapping[k], list)
|
|
1532
1516
|
}
|
|
1533
1517
|
)
|
|
1534
1518
|
|
|
1535
|
-
qp, _ = self.build_query_string(
|
|
1519
|
+
qp, _ = self.build_query_string(collection, keywords)
|
|
1536
1520
|
|
|
1537
1521
|
for query_param, query_value in qp.items():
|
|
1538
1522
|
if (
|
|
1539
1523
|
query_param
|
|
1540
|
-
in self.config.products.get(
|
|
1524
|
+
in self.config.products.get(collection, {}).get(
|
|
1541
1525
|
"specific_qssearch", {"parameters": []}
|
|
1542
1526
|
)["parameters"]
|
|
1543
1527
|
):
|
|
@@ -1545,20 +1529,20 @@ class PostJsonSearch(QueryStringSearch):
|
|
|
1545
1529
|
plugin_config_backup = yaml.dump(self.config)
|
|
1546
1530
|
|
|
1547
1531
|
self.config.api_endpoint = query_value
|
|
1548
|
-
self.config.products[
|
|
1532
|
+
self.config.products[collection][
|
|
1549
1533
|
"metadata_mapping"
|
|
1550
1534
|
] = mtd_cfg_as_conversion_and_querypath(
|
|
1551
|
-
self.config.products[
|
|
1535
|
+
self.config.products[collection]["specific_qssearch"][
|
|
1552
1536
|
"metadata_mapping"
|
|
1553
1537
|
]
|
|
1554
1538
|
)
|
|
1555
|
-
self.config.results_entry = self.config.products[
|
|
1539
|
+
self.config.results_entry = self.config.products[collection][
|
|
1556
1540
|
"specific_qssearch"
|
|
1557
1541
|
]["results_entry"]
|
|
1558
|
-
self.config.
|
|
1542
|
+
self.config._collection = self.config.products[collection][
|
|
1559
1543
|
"specific_qssearch"
|
|
1560
|
-
].get("
|
|
1561
|
-
self.config.merge_responses = self.config.products[
|
|
1544
|
+
].get("_collection")
|
|
1545
|
+
self.config.merge_responses = self.config.products[collection][
|
|
1562
1546
|
"specific_qssearch"
|
|
1563
1547
|
].get("merge_responses")
|
|
1564
1548
|
|
|
@@ -1585,14 +1569,14 @@ class PostJsonSearch(QueryStringSearch):
|
|
|
1585
1569
|
# If we were not able to build query params but have queryable search criteria,
|
|
1586
1570
|
# this means the provider does not support the search criteria given. If so,
|
|
1587
1571
|
# stop searching right away
|
|
1588
|
-
|
|
1572
|
+
collection_metadata_mapping = dict(
|
|
1589
1573
|
self.config.metadata_mapping,
|
|
1590
|
-
**prep.
|
|
1574
|
+
**prep.collection_def_params.get("metadata_mapping", {}),
|
|
1591
1575
|
)
|
|
1592
1576
|
if not qp and any(
|
|
1593
1577
|
k
|
|
1594
1578
|
for k in keywords.keys()
|
|
1595
|
-
if isinstance(
|
|
1579
|
+
if isinstance(collection_metadata_mapping.get(k), list)
|
|
1596
1580
|
):
|
|
1597
1581
|
return ([], 0) if prep.count else ([], None)
|
|
1598
1582
|
prep.query_params = dict(qp, **sort_by_qp)
|
|
@@ -1608,7 +1592,7 @@ class PostJsonSearch(QueryStringSearch):
|
|
|
1608
1592
|
|
|
1609
1593
|
raw_search_result = RawSearchResult(provider_results)
|
|
1610
1594
|
raw_search_result.query_params = prep.query_params
|
|
1611
|
-
raw_search_result.
|
|
1595
|
+
raw_search_result.collection_def_params = prep.collection_def_params
|
|
1612
1596
|
|
|
1613
1597
|
eo_products = self.normalize_results(raw_search_result, **kwargs)
|
|
1614
1598
|
return eo_products, total_items
|
|
@@ -1619,31 +1603,31 @@ class PostJsonSearch(QueryStringSearch):
|
|
|
1619
1603
|
"""Build EOProducts from provider results"""
|
|
1620
1604
|
normalized = super().normalize_results(results, **kwargs)
|
|
1621
1605
|
for product in normalized:
|
|
1622
|
-
if "
|
|
1623
|
-
decoded_link = unquote(product.properties["
|
|
1606
|
+
if "eodag:download_link" in product.properties:
|
|
1607
|
+
decoded_link = unquote(product.properties["eodag:download_link"])
|
|
1624
1608
|
if decoded_link[0] == "{": # not a url but a dict
|
|
1625
1609
|
default_values = deepcopy(
|
|
1626
|
-
self.config.products.get(product.
|
|
1610
|
+
self.config.products.get(product.collection, {})
|
|
1627
1611
|
)
|
|
1628
1612
|
default_values.pop("metadata_mapping", None)
|
|
1629
1613
|
searched_values = orjson.loads(decoded_link)
|
|
1630
1614
|
_dc_qs = orjson.dumps(
|
|
1631
1615
|
format_query_params(
|
|
1632
|
-
product.
|
|
1616
|
+
product.collection,
|
|
1633
1617
|
self.config,
|
|
1634
1618
|
{**default_values, **searched_values},
|
|
1635
1619
|
)
|
|
1636
1620
|
)
|
|
1637
1621
|
product.properties["_dc_qs"] = quote_plus(_dc_qs)
|
|
1638
1622
|
|
|
1639
|
-
# workaround to add
|
|
1623
|
+
# workaround to add collection to wekeo cmems order links
|
|
1640
1624
|
if (
|
|
1641
|
-
"
|
|
1642
|
-
and "
|
|
1625
|
+
"eodag:order_link" in product.properties
|
|
1626
|
+
and "collection" in product.properties["eodag:order_link"]
|
|
1643
1627
|
):
|
|
1644
|
-
product.properties["
|
|
1645
|
-
"
|
|
1646
|
-
].replace("
|
|
1628
|
+
product.properties["eodag:order_link"] = product.properties[
|
|
1629
|
+
"eodag:order_link"
|
|
1630
|
+
].replace("collection", product.collection)
|
|
1647
1631
|
return normalized
|
|
1648
1632
|
|
|
1649
1633
|
def collect_search_urls(
|
|
@@ -1668,10 +1652,10 @@ class PostJsonSearch(QueryStringSearch):
|
|
|
1668
1652
|
auth_conf_dict = getattr(prep.auth_plugin.config, "credentials", {})
|
|
1669
1653
|
else:
|
|
1670
1654
|
auth_conf_dict = {}
|
|
1671
|
-
for
|
|
1655
|
+
for _collection in self.get_provider_collections(prep, **kwargs) or (None,):
|
|
1672
1656
|
try:
|
|
1673
1657
|
search_endpoint: str = self.config.api_endpoint.rstrip("/").format(
|
|
1674
|
-
**dict(
|
|
1658
|
+
**dict(_collection=_collection, **auth_conf_dict)
|
|
1675
1659
|
)
|
|
1676
1660
|
except KeyError as e:
|
|
1677
1661
|
provider = prep.auth_plugin.provider if prep.auth_plugin else ""
|
|
@@ -1683,7 +1667,7 @@ class PostJsonSearch(QueryStringSearch):
|
|
|
1683
1667
|
if count:
|
|
1684
1668
|
count_endpoint = self.config.pagination.get(
|
|
1685
1669
|
"count_endpoint", ""
|
|
1686
|
-
).format(**dict(
|
|
1670
|
+
).format(**dict(_collection=_collection, **auth_conf_dict))
|
|
1687
1671
|
if count_endpoint:
|
|
1688
1672
|
_total_results = self.count_hits(
|
|
1689
1673
|
count_endpoint, result_type=self.config.result_type
|
|
@@ -1801,7 +1785,7 @@ class StacSearch(PostJsonSearch):
|
|
|
1801
1785
|
have to be overwritten. If certain functionalities are not available, their configuration
|
|
1802
1786
|
parameters have to be overwritten with ``null``. E.g. if there is no queryables endpoint,
|
|
1803
1787
|
the :attr:`~eodag.config.PluginConfig.DiscoverQueryables.fetch_url` and
|
|
1804
|
-
:attr:`~eodag.config.PluginConfig.DiscoverQueryables.
|
|
1788
|
+
:attr:`~eodag.config.PluginConfig.DiscoverQueryables.collection_fetch_url` in the
|
|
1805
1789
|
:attr:`~eodag.config.PluginConfig.discover_queryables` config have to be set to ``null``.
|
|
1806
1790
|
"""
|
|
1807
1791
|
|
|
@@ -1815,22 +1799,19 @@ class StacSearch(PostJsonSearch):
|
|
|
1815
1799
|
self.config.results_entry = results_entry
|
|
1816
1800
|
|
|
1817
1801
|
def build_query_string(
|
|
1818
|
-
self,
|
|
1802
|
+
self, collection: str, query_dict: dict[str, Any]
|
|
1819
1803
|
) -> tuple[dict[str, Any], str]:
|
|
1820
1804
|
"""Build The query string using the search parameters"""
|
|
1821
1805
|
logger.debug("Building the query string that will be used for search")
|
|
1822
1806
|
|
|
1823
1807
|
# handle opened time intervals
|
|
1824
|
-
if any(
|
|
1825
|
-
|
|
1826
|
-
|
|
1827
|
-
):
|
|
1828
|
-
query_dict.setdefault("startTimeFromAscendingNode", "..")
|
|
1829
|
-
query_dict.setdefault("completionTimeFromAscendingNode", "..")
|
|
1808
|
+
if any(q in query_dict for q in ("start_datetime", "end_datetime")):
|
|
1809
|
+
query_dict.setdefault("start_datetime", "..")
|
|
1810
|
+
query_dict.setdefault("end_datetime", "..")
|
|
1830
1811
|
|
|
1831
|
-
error_context = f"
|
|
1812
|
+
error_context = f"Collection: {collection} / provider : {self.provider}"
|
|
1832
1813
|
query_params = format_query_params(
|
|
1833
|
-
|
|
1814
|
+
collection, self.config, query_dict, error_context
|
|
1834
1815
|
)
|
|
1835
1816
|
|
|
1836
1817
|
# Build the final query string, in one go without quoting it
|
|
@@ -1848,41 +1829,38 @@ class StacSearch(PostJsonSearch):
|
|
|
1848
1829
|
) -> Optional[dict[str, Annotated[Any, FieldInfo]]]:
|
|
1849
1830
|
"""Fetch queryables list from provider using `discover_queryables` conf
|
|
1850
1831
|
|
|
1851
|
-
:param kwargs: additional filters for queryables (`
|
|
1832
|
+
:param kwargs: additional filters for queryables (`collection` and other search
|
|
1852
1833
|
arguments)
|
|
1853
1834
|
:returns: fetched queryable parameters dict
|
|
1854
1835
|
"""
|
|
1855
1836
|
if (
|
|
1856
1837
|
not self.config.discover_queryables["fetch_url"]
|
|
1857
|
-
and not self.config.discover_queryables["
|
|
1838
|
+
and not self.config.discover_queryables["collection_fetch_url"]
|
|
1858
1839
|
):
|
|
1859
1840
|
raise NotImplementedError()
|
|
1860
1841
|
|
|
1861
|
-
|
|
1862
|
-
|
|
1863
|
-
self.config.products.get(
|
|
1864
|
-
if
|
|
1842
|
+
collection = kwargs.get("collection")
|
|
1843
|
+
provider_collection = (
|
|
1844
|
+
self.config.products.get(collection, {}).get("_collection", collection)
|
|
1845
|
+
if collection
|
|
1865
1846
|
else None
|
|
1866
1847
|
)
|
|
1867
1848
|
if (
|
|
1868
|
-
|
|
1869
|
-
and not self.config.discover_queryables["
|
|
1849
|
+
provider_collection
|
|
1850
|
+
and not self.config.discover_queryables["collection_fetch_url"]
|
|
1870
1851
|
):
|
|
1871
1852
|
raise NotImplementedError(
|
|
1872
|
-
f"Cannot fetch queryables for a specific
|
|
1853
|
+
f"Cannot fetch queryables for a specific collection with {self.provider}"
|
|
1873
1854
|
)
|
|
1874
|
-
if
|
|
1875
|
-
not provider_product_type
|
|
1876
|
-
and not self.config.discover_queryables["fetch_url"]
|
|
1877
|
-
):
|
|
1855
|
+
if not provider_collection and not self.config.discover_queryables["fetch_url"]:
|
|
1878
1856
|
raise ValidationError(
|
|
1879
|
-
f"Cannot fetch global queryables for {self.provider}. A
|
|
1857
|
+
f"Cannot fetch global queryables for {self.provider}. A collection must be specified"
|
|
1880
1858
|
)
|
|
1881
1859
|
|
|
1882
1860
|
try:
|
|
1883
1861
|
unparsed_fetch_url = (
|
|
1884
|
-
self.config.discover_queryables["
|
|
1885
|
-
if
|
|
1862
|
+
self.config.discover_queryables["collection_fetch_url"]
|
|
1863
|
+
if provider_collection
|
|
1886
1864
|
else self.config.discover_queryables["fetch_url"]
|
|
1887
1865
|
)
|
|
1888
1866
|
if unparsed_fetch_url is None:
|
|
@@ -1891,7 +1869,7 @@ class StacSearch(PostJsonSearch):
|
|
|
1891
1869
|
)
|
|
1892
1870
|
|
|
1893
1871
|
fetch_url = unparsed_fetch_url.format(
|
|
1894
|
-
|
|
1872
|
+
provider_collection=provider_collection,
|
|
1895
1873
|
**self.config.__dict__,
|
|
1896
1874
|
)
|
|
1897
1875
|
auth = (
|
|
@@ -1939,40 +1917,43 @@ class StacSearch(PostJsonSearch):
|
|
|
1939
1917
|
)
|
|
1940
1918
|
except IndexError:
|
|
1941
1919
|
logger.info(
|
|
1942
|
-
"No queryable found for %s on %s",
|
|
1920
|
+
"No queryable found for %s on %s", collection, self.provider
|
|
1943
1921
|
)
|
|
1944
1922
|
return None
|
|
1945
1923
|
# convert json results to pydantic model fields
|
|
1946
1924
|
field_definitions: dict[str, Any] = dict()
|
|
1947
|
-
|
|
1948
|
-
"start_datetime": "start",
|
|
1949
|
-
"end_datetime": "end",
|
|
1950
|
-
"datetime": None,
|
|
1951
|
-
"bbox": "geom",
|
|
1952
|
-
}
|
|
1925
|
+
eodag_queryables_and_defaults: list[tuple[str, Any]] = []
|
|
1953
1926
|
for json_param, json_mtd in json_queryables.items():
|
|
1954
|
-
param =
|
|
1955
|
-
json_param,
|
|
1956
|
-
|
|
1957
|
-
|
|
1958
|
-
|
|
1959
|
-
or json_param,
|
|
1960
|
-
)
|
|
1961
|
-
if param is None:
|
|
1927
|
+
param = get_queryable_from_provider(
|
|
1928
|
+
json_param, self.get_metadata_mapping(collection)
|
|
1929
|
+
) or Queryables.get_queryable_from_alias(json_param)
|
|
1930
|
+
# do not expose internal parameters, neither datetime
|
|
1931
|
+
if param == "datetime" or param.startswith("_"):
|
|
1962
1932
|
continue
|
|
1963
1933
|
|
|
1964
1934
|
default = kwargs.get(param, json_mtd.get("default"))
|
|
1935
|
+
|
|
1936
|
+
if param in Queryables.model_fields:
|
|
1937
|
+
# use eodag queryable as default
|
|
1938
|
+
eodag_queryables_and_defaults += [(param, default)]
|
|
1939
|
+
continue
|
|
1940
|
+
|
|
1941
|
+
# convert provider json field definition to python
|
|
1942
|
+
default = kwargs.get(param, json_mtd.get("default"))
|
|
1965
1943
|
annotated_def = json_field_definition_to_python(
|
|
1966
1944
|
json_mtd, default_value=default
|
|
1967
1945
|
)
|
|
1968
|
-
|
|
1946
|
+
field_definition = get_args(annotated_def)
|
|
1947
|
+
field_definitions[param] = field_definition
|
|
1969
1948
|
|
|
1970
1949
|
python_queryables = create_model("m", **field_definitions).model_fields
|
|
1971
|
-
geom_queryable = python_queryables.pop("geometry", None)
|
|
1972
|
-
if geom_queryable:
|
|
1973
|
-
python_queryables["geom"] = Queryables.model_fields["geom"]
|
|
1974
1950
|
|
|
1975
1951
|
queryables_dict = model_fields_to_annotated(python_queryables)
|
|
1952
|
+
|
|
1953
|
+
# append eodag queryables
|
|
1954
|
+
for param, default in eodag_queryables_and_defaults:
|
|
1955
|
+
queryables_dict[param] = Queryables.get_with_default(param, default)
|
|
1956
|
+
|
|
1976
1957
|
# append "datetime" as "start" & "end" if needed
|
|
1977
1958
|
if "datetime" in json_queryables:
|
|
1978
1959
|
eodag_queryables = copy_deepcopy(
|
|
@@ -1993,7 +1974,7 @@ class WekeoSearch(StacSearch, PostJsonSearch):
|
|
|
1993
1974
|
PostJsonSearch.__init__(self, provider, config)
|
|
1994
1975
|
|
|
1995
1976
|
def build_query_string(
|
|
1996
|
-
self,
|
|
1977
|
+
self, collection: str, query_dict: dict[str, Any]
|
|
1997
1978
|
) -> tuple[dict[str, Any], str]:
|
|
1998
1979
|
"""Build The query string using the search parameters"""
|
|
1999
|
-
return PostJsonSearch.build_query_string(self,
|
|
1980
|
+
return PostJsonSearch.build_query_string(self, collection, query_dict)
|