eodag 3.0.0b3__py3-none-any.whl → 3.0.1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- eodag/api/core.py +189 -125
- eodag/api/product/metadata_mapping.py +12 -3
- eodag/api/search_result.py +29 -3
- eodag/cli.py +35 -19
- eodag/config.py +412 -116
- eodag/plugins/apis/base.py +10 -4
- eodag/plugins/apis/ecmwf.py +14 -4
- eodag/plugins/apis/usgs.py +25 -2
- eodag/plugins/authentication/aws_auth.py +14 -5
- eodag/plugins/authentication/base.py +10 -1
- eodag/plugins/authentication/generic.py +14 -3
- eodag/plugins/authentication/header.py +12 -4
- eodag/plugins/authentication/keycloak.py +41 -22
- eodag/plugins/authentication/oauth.py +11 -1
- eodag/plugins/authentication/openid_connect.py +178 -163
- eodag/plugins/authentication/qsauth.py +12 -4
- eodag/plugins/authentication/sas_auth.py +19 -2
- eodag/plugins/authentication/token.py +57 -10
- eodag/plugins/authentication/token_exchange.py +19 -19
- eodag/plugins/crunch/base.py +4 -1
- eodag/plugins/crunch/filter_date.py +5 -2
- eodag/plugins/crunch/filter_latest_intersect.py +5 -4
- eodag/plugins/crunch/filter_latest_tpl_name.py +1 -1
- eodag/plugins/crunch/filter_overlap.py +5 -7
- eodag/plugins/crunch/filter_property.py +4 -3
- eodag/plugins/download/aws.py +39 -22
- eodag/plugins/download/base.py +11 -11
- eodag/plugins/download/creodias_s3.py +11 -2
- eodag/plugins/download/http.py +86 -52
- eodag/plugins/download/s3rest.py +20 -18
- eodag/plugins/manager.py +168 -23
- eodag/plugins/search/base.py +33 -14
- eodag/plugins/search/build_search_result.py +55 -51
- eodag/plugins/search/cop_marine.py +112 -29
- eodag/plugins/search/creodias_s3.py +20 -5
- eodag/plugins/search/csw.py +41 -1
- eodag/plugins/search/data_request_search.py +109 -9
- eodag/plugins/search/qssearch.py +532 -152
- eodag/plugins/search/static_stac_search.py +20 -21
- eodag/resources/ext_product_types.json +1 -1
- eodag/resources/product_types.yml +187 -56
- eodag/resources/providers.yml +1610 -1701
- eodag/resources/stac.yml +3 -163
- eodag/resources/user_conf_template.yml +112 -97
- eodag/rest/config.py +1 -2
- eodag/rest/constants.py +0 -1
- eodag/rest/core.py +61 -51
- eodag/rest/errors.py +181 -0
- eodag/rest/server.py +24 -325
- eodag/rest/stac.py +93 -544
- eodag/rest/types/eodag_search.py +13 -8
- eodag/rest/types/queryables.py +1 -2
- eodag/rest/types/stac_search.py +11 -2
- eodag/types/__init__.py +15 -3
- eodag/types/download_args.py +1 -1
- eodag/types/queryables.py +1 -2
- eodag/types/search_args.py +3 -3
- eodag/utils/__init__.py +77 -57
- eodag/utils/exceptions.py +23 -9
- eodag/utils/logging.py +37 -77
- eodag/utils/requests.py +1 -3
- eodag/utils/stac_reader.py +1 -1
- {eodag-3.0.0b3.dist-info → eodag-3.0.1.dist-info}/METADATA +11 -12
- eodag-3.0.1.dist-info/RECORD +109 -0
- {eodag-3.0.0b3.dist-info → eodag-3.0.1.dist-info}/WHEEL +1 -1
- {eodag-3.0.0b3.dist-info → eodag-3.0.1.dist-info}/entry_points.txt +1 -0
- eodag/resources/constraints/climate-dt.json +0 -13
- eodag/resources/constraints/extremes-dt.json +0 -8
- eodag-3.0.0b3.dist-info/RECORD +0 -110
- {eodag-3.0.0b3.dist-info → eodag-3.0.1.dist-info}/LICENSE +0 -0
- {eodag-3.0.0b3.dist-info → eodag-3.0.1.dist-info}/top_level.txt +0 -0
eodag/plugins/search/base.py
CHANGED
|
@@ -18,7 +18,7 @@
|
|
|
18
18
|
from __future__ import annotations
|
|
19
19
|
|
|
20
20
|
import logging
|
|
21
|
-
from typing import TYPE_CHECKING
|
|
21
|
+
from typing import TYPE_CHECKING, Annotated, get_args
|
|
22
22
|
|
|
23
23
|
import orjson
|
|
24
24
|
from pydantic.fields import Field, FieldInfo
|
|
@@ -35,11 +35,9 @@ from eodag.types.queryables import Queryables
|
|
|
35
35
|
from eodag.types.search_args import SortByList
|
|
36
36
|
from eodag.utils import (
|
|
37
37
|
GENERIC_PRODUCT_TYPE,
|
|
38
|
-
Annotated,
|
|
39
38
|
copy_deepcopy,
|
|
40
39
|
deepcopy,
|
|
41
40
|
format_dict_items,
|
|
42
|
-
get_args,
|
|
43
41
|
update_nested_dict,
|
|
44
42
|
)
|
|
45
43
|
from eodag.utils.exceptions import ValidationError
|
|
@@ -95,9 +93,9 @@ class Search(PluginTopic):
|
|
|
95
93
|
) -> Tuple[List[EOProduct], Optional[int]]:
|
|
96
94
|
"""Implementation of how the products must be searched goes here.
|
|
97
95
|
|
|
98
|
-
This method must return a tuple with (1) a list of
|
|
99
|
-
which will be processed by a Download plugin (2) and the total number of
|
|
100
|
-
the search criteria. If ``prep.count`` is False, the second element returned must be ``None``.
|
|
96
|
+
This method must return a tuple with (1) a list of :class:`~eodag.api.product._product.EOProduct` instances
|
|
97
|
+
which will be processed by a :class:`~eodag.plugins.download.base.Download` plugin (2) and the total number of
|
|
98
|
+
products matching the search criteria. If ``prep.count`` is False, the second element returned must be ``None``.
|
|
101
99
|
"""
|
|
102
100
|
raise NotImplementedError("A Search plugin must implement a method named query")
|
|
103
101
|
|
|
@@ -108,9 +106,9 @@ class Search(PluginTopic):
|
|
|
108
106
|
def discover_queryables(
|
|
109
107
|
self, **kwargs: Any
|
|
110
108
|
) -> Optional[Dict[str, Annotated[Any, FieldInfo]]]:
|
|
111
|
-
"""Fetch queryables list from provider using
|
|
109
|
+
"""Fetch queryables list from provider using :attr:`~eodag.config.PluginConfig.discover_queryables` conf
|
|
112
110
|
|
|
113
|
-
:param kwargs: additional filters for queryables (
|
|
111
|
+
:param kwargs: additional filters for queryables (``productType`` and other search
|
|
114
112
|
arguments)
|
|
115
113
|
:returns: fetched queryable parameters dict
|
|
116
114
|
"""
|
|
@@ -179,6 +177,27 @@ class Search(PluginTopic):
|
|
|
179
177
|
else:
|
|
180
178
|
return {}
|
|
181
179
|
|
|
180
|
+
def get_product_type_cfg_value(self, key: str, default: Any = None) -> Any:
|
|
181
|
+
"""
|
|
182
|
+
Get the value of a configuration option specific to the current product type.
|
|
183
|
+
|
|
184
|
+
This method retrieves the value of a configuration option from the
|
|
185
|
+
``product_type_config`` attribute. If the option is not found, the provided
|
|
186
|
+
default value is returned.
|
|
187
|
+
|
|
188
|
+
:param key: The configuration option key.
|
|
189
|
+
:type key: str
|
|
190
|
+
:param default: The default value to be returned if the option is not found (default is None).
|
|
191
|
+
:type default: Any
|
|
192
|
+
|
|
193
|
+
:return: The value of the specified configuration option or the default value.
|
|
194
|
+
:rtype: Any
|
|
195
|
+
"""
|
|
196
|
+
product_type_cfg = getattr(self.config, "product_type_config", {})
|
|
197
|
+
non_none_cfg = {k: v for k, v in product_type_cfg.items() if v}
|
|
198
|
+
|
|
199
|
+
return non_none_cfg.get(key, default)
|
|
200
|
+
|
|
182
201
|
def get_metadata_mapping(
|
|
183
202
|
self, product_type: Optional[str] = None
|
|
184
203
|
) -> Dict[str, Union[str, List[str]]]:
|
|
@@ -194,10 +213,10 @@ class Search(PluginTopic):
|
|
|
194
213
|
return self.config.metadata_mapping
|
|
195
214
|
|
|
196
215
|
def get_sort_by_arg(self, kwargs: Dict[str, Any]) -> Optional[SortByList]:
|
|
197
|
-
"""Extract the
|
|
216
|
+
"""Extract the ``sort_by`` argument from the kwargs or the provider default sort configuration
|
|
198
217
|
|
|
199
218
|
:param kwargs: Search arguments
|
|
200
|
-
:returns: The
|
|
219
|
+
:returns: The ``sort_by`` argument from the kwargs or the provider default sort configuration
|
|
201
220
|
"""
|
|
202
221
|
# remove "sort_by" from search args if exists because it is not part of metadata mapping,
|
|
203
222
|
# it will complete the query string or body once metadata mapping will be done
|
|
@@ -216,16 +235,16 @@ class Search(PluginTopic):
|
|
|
216
235
|
self, sort_by_arg: SortByList
|
|
217
236
|
) -> Tuple[str, Dict[str, List[Dict[str, str]]]]:
|
|
218
237
|
"""Build the sorting part of the query string or body by transforming
|
|
219
|
-
the
|
|
238
|
+
the ``sort_by`` argument into a provider-specific string or dictionary
|
|
220
239
|
|
|
221
|
-
:param sort_by_arg: the
|
|
222
|
-
:returns: The
|
|
240
|
+
:param sort_by_arg: the ``sort_by`` argument in EODAG format
|
|
241
|
+
:returns: The ``sort_by`` argument in provider-specific format
|
|
223
242
|
"""
|
|
224
243
|
if not hasattr(self.config, "sort"):
|
|
225
244
|
raise ValidationError(f"{self.provider} does not support sorting feature")
|
|
226
245
|
# TODO: remove this code block when search args model validation is embeded
|
|
227
246
|
# remove duplicates
|
|
228
|
-
sort_by_arg = list(
|
|
247
|
+
sort_by_arg = list(dict.fromkeys(sort_by_arg))
|
|
229
248
|
|
|
230
249
|
sort_by_qs: str = ""
|
|
231
250
|
sort_by_qp: Dict[str, Any] = {}
|
|
@@ -20,16 +20,27 @@ from __future__ import annotations
|
|
|
20
20
|
import hashlib
|
|
21
21
|
import logging
|
|
22
22
|
from datetime import datetime, timedelta, timezone
|
|
23
|
-
from typing import
|
|
23
|
+
from typing import (
|
|
24
|
+
TYPE_CHECKING,
|
|
25
|
+
Annotated,
|
|
26
|
+
Any,
|
|
27
|
+
Dict,
|
|
28
|
+
List,
|
|
29
|
+
Optional,
|
|
30
|
+
Set,
|
|
31
|
+
Tuple,
|
|
32
|
+
cast,
|
|
33
|
+
get_args,
|
|
34
|
+
)
|
|
24
35
|
from urllib.parse import quote_plus, unquote_plus
|
|
25
36
|
|
|
26
37
|
import geojson
|
|
27
38
|
import orjson
|
|
28
39
|
from dateutil.parser import isoparse
|
|
40
|
+
from dateutil.tz import tzutc
|
|
29
41
|
from jsonpath_ng import Child, Fields, Root
|
|
30
42
|
from pydantic import create_model
|
|
31
43
|
from pydantic.fields import FieldInfo
|
|
32
|
-
from typing_extensions import get_args
|
|
33
44
|
|
|
34
45
|
from eodag.api.product import EOProduct
|
|
35
46
|
from eodag.api.product.metadata_mapping import (
|
|
@@ -47,7 +58,6 @@ from eodag.types import json_field_definition_to_python, model_fields_to_annotat
|
|
|
47
58
|
from eodag.types.queryables import CommonQueryables
|
|
48
59
|
from eodag.utils import (
|
|
49
60
|
DEFAULT_MISSION_START_DATE,
|
|
50
|
-
Annotated,
|
|
51
61
|
deepcopy,
|
|
52
62
|
dict_items_recursive_sort,
|
|
53
63
|
get_geometry_from_various,
|
|
@@ -71,21 +81,16 @@ class BuildPostSearchResult(PostJsonSearch):
|
|
|
71
81
|
performs a POST request and uses its result to build a single :class:`~eodag.api.search_result.SearchResult`
|
|
72
82
|
object.
|
|
73
83
|
|
|
74
|
-
The available configuration parameters
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
- **api_endpoint**: (mandatory) The endpoint of the provider's search interface
|
|
84
|
+
The available configuration parameters are inherited from parent classes
|
|
85
|
+
(:class:`~eodag.plugins.search.qssearch.PostJsonSearch` and
|
|
86
|
+
:class:`~eodag.plugins.search.qssearch.QueryStringSearch`), with particularly for this plugin:
|
|
78
87
|
|
|
79
|
-
|
|
80
|
-
|
|
88
|
+
:param provider: provider name
|
|
89
|
+
:param config: Search plugin configuration:
|
|
81
90
|
|
|
82
|
-
|
|
83
|
-
|
|
84
|
-
formatted like `{{"foo":"bar"}}` because it will be passed to a `.format()`
|
|
85
|
-
method before being loaded as json.
|
|
91
|
+
* :attr:`~eodag.config.PluginConfig.remove_from_query` (``List[str]``): List of parameters
|
|
92
|
+
used to parse metadata but that must not be included to the query
|
|
86
93
|
|
|
87
|
-
:param provider: An eodag providers configuration dictionary
|
|
88
|
-
:param config: Path to the user configuration file
|
|
89
94
|
"""
|
|
90
95
|
|
|
91
96
|
def count_hits(
|
|
@@ -110,8 +115,8 @@ class BuildPostSearchResult(PostJsonSearch):
|
|
|
110
115
|
prep.url = prep.search_urls[0]
|
|
111
116
|
prep.info_message = f"Sending search request: {prep.url}"
|
|
112
117
|
prep.exception_message = (
|
|
113
|
-
f"Skipping error while searching for {self.provider}
|
|
114
|
-
f"{self.__class__.__name__} instance
|
|
118
|
+
f"Skipping error while searching for {self.provider}"
|
|
119
|
+
f" {self.__class__.__name__} instance"
|
|
115
120
|
)
|
|
116
121
|
response = self._request(prep)
|
|
117
122
|
|
|
@@ -178,7 +183,7 @@ class BuildPostSearchResult(PostJsonSearch):
|
|
|
178
183
|
result.update(results.product_type_def_params)
|
|
179
184
|
result = dict(result, **{k: v for k, v in kwargs.items() if v is not None})
|
|
180
185
|
|
|
181
|
-
# parse
|
|
186
|
+
# parse properties
|
|
182
187
|
parsed_properties = properties_from_json(
|
|
183
188
|
result,
|
|
184
189
|
self.config.metadata_mapping,
|
|
@@ -239,19 +244,18 @@ class BuildSearchResult(BuildPostSearchResult):
|
|
|
239
244
|
This plugin builds a single :class:`~eodag.api.search_result.SearchResult` object
|
|
240
245
|
using given query parameters as product properties.
|
|
241
246
|
|
|
242
|
-
The available configuration parameters inherits from parent classes
|
|
243
|
-
|
|
247
|
+
The available configuration parameters inherits from parent classes
|
|
248
|
+
(:class:`~eodag.plugins.search.build_search_result.BuildPostSearchResult`,
|
|
249
|
+
:class:`~eodag.plugins.search.qssearch.PostJsonSearch` and
|
|
250
|
+
:class:`~eodag.plugins.search.qssearch.QueryStringSearch`), with particularly for this plugin:
|
|
244
251
|
|
|
245
|
-
|
|
246
|
-
|
|
252
|
+
:param provider: provider name
|
|
253
|
+
:param config: Search plugin configuration:
|
|
247
254
|
|
|
248
|
-
|
|
249
|
-
not
|
|
255
|
+
* :attr:`~eodag.config.PluginConfig.end_date_excluded` (``bool``): Set to ``False`` if provider
|
|
256
|
+
does not include end date in the search request; In this case, if the end date is at midnight,
|
|
257
|
+
the previous day will be used. default: ``True``
|
|
250
258
|
|
|
251
|
-
- **constraints_file_url**: url of the constraint file used to build queryables
|
|
252
|
-
|
|
253
|
-
:param provider: An eodag providers configuration dictionary
|
|
254
|
-
:param config: Path to the user configuration file
|
|
255
259
|
"""
|
|
256
260
|
|
|
257
261
|
def __init__(self, provider: str, config: PluginConfig) -> None:
|
|
@@ -339,24 +343,6 @@ class BuildSearchResult(BuildPostSearchResult):
|
|
|
339
343
|
self, product_type=product_type, **available_properties
|
|
340
344
|
)
|
|
341
345
|
|
|
342
|
-
def get_product_type_cfg(self, key: str, default: Any = None) -> Any:
|
|
343
|
-
"""
|
|
344
|
-
Get the value of a configuration option specific to the current product type.
|
|
345
|
-
|
|
346
|
-
This method retrieves the value of a configuration option from the
|
|
347
|
-
`_product_type_config` attribute. If the option is not found, the provided
|
|
348
|
-
default value is returned.
|
|
349
|
-
|
|
350
|
-
:param key: The configuration option key.
|
|
351
|
-
:param default: The default value to be returned if the option is not found (default is None).
|
|
352
|
-
|
|
353
|
-
:return: The value of the specified configuration option or the default value.
|
|
354
|
-
"""
|
|
355
|
-
product_type_cfg = getattr(self.config, "product_type_config", {})
|
|
356
|
-
non_none_cfg = {k: v for k, v in product_type_cfg.items() if v}
|
|
357
|
-
|
|
358
|
-
return non_none_cfg.get(key, default)
|
|
359
|
-
|
|
360
346
|
def _preprocess_search_params(self, params: Dict[str, Any]) -> None:
|
|
361
347
|
"""Preprocess search parameters before making a request to the CDS API.
|
|
362
348
|
|
|
@@ -397,7 +383,7 @@ class BuildSearchResult(BuildPostSearchResult):
|
|
|
397
383
|
|
|
398
384
|
# dates
|
|
399
385
|
mission_start_dt = datetime.fromisoformat(
|
|
400
|
-
self.
|
|
386
|
+
self.get_product_type_cfg_value(
|
|
401
387
|
"missionStartDate", DEFAULT_MISSION_START_DATE
|
|
402
388
|
).replace(
|
|
403
389
|
"Z", "+00:00"
|
|
@@ -423,11 +409,29 @@ class BuildSearchResult(BuildPostSearchResult):
|
|
|
423
409
|
"completionTimeFromAscendingNode", default_end_str
|
|
424
410
|
)
|
|
425
411
|
|
|
426
|
-
#
|
|
412
|
+
# adapt end date if it is midnight
|
|
427
413
|
end_date_excluded = getattr(self.config, "end_date_excluded", True)
|
|
428
|
-
|
|
429
|
-
|
|
430
|
-
|
|
414
|
+
is_datetime = True
|
|
415
|
+
try:
|
|
416
|
+
end_date = datetime.strptime(
|
|
417
|
+
params["completionTimeFromAscendingNode"], "%Y-%m-%dT%H:%M:%SZ"
|
|
418
|
+
)
|
|
419
|
+
end_date = end_date.replace(tzinfo=tzutc())
|
|
420
|
+
except ValueError:
|
|
421
|
+
try:
|
|
422
|
+
end_date = datetime.strptime(
|
|
423
|
+
params["completionTimeFromAscendingNode"], "%Y-%m-%dT%H:%M:%S.%fZ"
|
|
424
|
+
)
|
|
425
|
+
end_date = end_date.replace(tzinfo=tzutc())
|
|
426
|
+
except ValueError:
|
|
427
|
+
end_date = isoparse(params["completionTimeFromAscendingNode"])
|
|
428
|
+
is_datetime = False
|
|
429
|
+
start_date = isoparse(params["startTimeFromAscendingNode"])
|
|
430
|
+
if (
|
|
431
|
+
not end_date_excluded
|
|
432
|
+
and is_datetime
|
|
433
|
+
and end_date > start_date
|
|
434
|
+
and end_date == end_date.replace(hour=0, minute=0, second=0, microsecond=0)
|
|
431
435
|
):
|
|
432
436
|
end_date += timedelta(days=-1)
|
|
433
437
|
params["completionTimeFromAscendingNode"] = end_date.isoformat()
|
|
@@ -19,6 +19,7 @@ from __future__ import annotations
|
|
|
19
19
|
|
|
20
20
|
import copy
|
|
21
21
|
import logging
|
|
22
|
+
import os
|
|
22
23
|
import re
|
|
23
24
|
from datetime import datetime
|
|
24
25
|
from typing import TYPE_CHECKING, Any, Dict, List, Optional, Tuple, cast
|
|
@@ -37,7 +38,7 @@ from eodag.config import PluginConfig
|
|
|
37
38
|
from eodag.plugins.search import PreparedSearch
|
|
38
39
|
from eodag.plugins.search.static_stac_search import StaticStacSearch
|
|
39
40
|
from eodag.utils import get_bucket_name_and_prefix
|
|
40
|
-
from eodag.utils.exceptions import UnsupportedProductType, ValidationError
|
|
41
|
+
from eodag.utils.exceptions import RequestError, UnsupportedProductType, ValidationError
|
|
41
42
|
|
|
42
43
|
if TYPE_CHECKING:
|
|
43
44
|
from mypy_boto3_s3 import S3Client
|
|
@@ -67,6 +68,21 @@ def _get_date_from_yyyymmdd(date_str: str, item_key: str) -> Optional[datetime]:
|
|
|
67
68
|
return date
|
|
68
69
|
|
|
69
70
|
|
|
71
|
+
def _get_dates_from_dataset_data(
|
|
72
|
+
dataset_item: Dict[str, Any]
|
|
73
|
+
) -> Optional[Dict[str, str]]:
|
|
74
|
+
dates = {}
|
|
75
|
+
if "start_datetime" in dataset_item["properties"]:
|
|
76
|
+
dates["start"] = dataset_item["properties"]["start_datetime"]
|
|
77
|
+
dates["end"] = dataset_item["properties"]["end_datetime"]
|
|
78
|
+
elif "datetime" in dataset_item["properties"]:
|
|
79
|
+
dates["start"] = dataset_item["properties"]["datetime"]
|
|
80
|
+
dates["end"] = dataset_item["properties"]["datetime"]
|
|
81
|
+
else:
|
|
82
|
+
return None
|
|
83
|
+
return dates
|
|
84
|
+
|
|
85
|
+
|
|
70
86
|
def _get_s3_client(endpoint_url: str) -> S3Client:
|
|
71
87
|
s3_session = boto3.Session()
|
|
72
88
|
return s3_session.client(
|
|
@@ -94,7 +110,21 @@ def _check_int_values_properties(properties: Dict[str, Any]):
|
|
|
94
110
|
|
|
95
111
|
|
|
96
112
|
class CopMarineSearch(StaticStacSearch):
|
|
97
|
-
"""class that implements search for the Copernicus Marine provider
|
|
113
|
+
"""class that implements search for the Copernicus Marine provider
|
|
114
|
+
|
|
115
|
+
It calls :meth:`~eodag.plugins.search.static_stac_search.StaticStacSearch.discover_product_types`
|
|
116
|
+
inherited from :class:`~eodag.plugins.search.static_stac_search.StaticStacSearch`
|
|
117
|
+
but for the actual search a special method which fetches the urls of the available products from an S3 storage and
|
|
118
|
+
filters them has been written.
|
|
119
|
+
|
|
120
|
+
The configuration parameters are inherited from the parent and grand-parent classes. The
|
|
121
|
+
:attr:`~eodag.config.PluginConfig.DiscoverMetadata.auto_discovery` parameter in the
|
|
122
|
+
:attr:`~eodag.config.PluginConfig.discover_metadata` section has to be set to ``false`` and the
|
|
123
|
+
:attr:`~eodag.config.PluginConfig.DiscoverQueryables.fetch_url` in the
|
|
124
|
+
:attr:`~eodag.config.PluginConfig.discover_queryables` queryables section has to be set to ``null`` to
|
|
125
|
+
overwrite the default config from the stac provider configuration because those functionalities
|
|
126
|
+
are not available.
|
|
127
|
+
"""
|
|
98
128
|
|
|
99
129
|
def __init__(self, provider: str, config: PluginConfig):
|
|
100
130
|
original_metadata_mapping = copy.deepcopy(config.metadata_mapping)
|
|
@@ -107,12 +137,10 @@ class CopMarineSearch(StaticStacSearch):
|
|
|
107
137
|
) -> Tuple[Dict[str, Any], List[Dict[str, Any]]]:
|
|
108
138
|
"""Fetch product type and associated datasets info"""
|
|
109
139
|
|
|
110
|
-
fetch_url = cast(
|
|
111
|
-
|
|
112
|
-
self.config.discover_product_types["fetch_url"].format(
|
|
113
|
-
**self.config.__dict__
|
|
114
|
-
),
|
|
140
|
+
fetch_url = cast(str, self.config.discover_product_types["fetch_url"]).format(
|
|
141
|
+
**self.config.__dict__
|
|
115
142
|
)
|
|
143
|
+
|
|
116
144
|
logger.debug("fetch data for collection %s", product_type)
|
|
117
145
|
provider_product_type = self.config.products.get(product_type, {}).get(
|
|
118
146
|
"productType", None
|
|
@@ -125,9 +153,14 @@ class CopMarineSearch(StaticStacSearch):
|
|
|
125
153
|
)
|
|
126
154
|
try:
|
|
127
155
|
collection_data = requests.get(collection_url).json()
|
|
128
|
-
except requests.RequestException:
|
|
156
|
+
except requests.RequestException as exc:
|
|
157
|
+
if exc.errno == 404:
|
|
158
|
+
logger.error("product %s not found", product_type)
|
|
159
|
+
raise UnsupportedProductType(product_type)
|
|
129
160
|
logger.error("data for product %s could not be fetched", product_type)
|
|
130
|
-
raise
|
|
161
|
+
raise RequestError.from_error(
|
|
162
|
+
exc, f"data for product {product_type} could not be fetched"
|
|
163
|
+
) from exc
|
|
131
164
|
|
|
132
165
|
datasets = []
|
|
133
166
|
for link in [li for li in collection_data["links"] if li["rel"] == "item"]:
|
|
@@ -170,7 +203,7 @@ class CopMarineSearch(StaticStacSearch):
|
|
|
170
203
|
use_dataset_dates: bool = False,
|
|
171
204
|
) -> Optional[EOProduct]:
|
|
172
205
|
|
|
173
|
-
item_id = item_key.split("/")[-1]
|
|
206
|
+
item_id = os.path.splitext(item_key.split("/")[-1])[0]
|
|
174
207
|
download_url = s3_url + "/" + item_key
|
|
175
208
|
properties = {
|
|
176
209
|
"id": item_id,
|
|
@@ -180,20 +213,16 @@ class CopMarineSearch(StaticStacSearch):
|
|
|
180
213
|
"dataset": dataset_item["id"],
|
|
181
214
|
}
|
|
182
215
|
if use_dataset_dates:
|
|
183
|
-
|
|
184
|
-
|
|
185
|
-
|
|
186
|
-
|
|
187
|
-
|
|
188
|
-
"end_datetime"
|
|
189
|
-
]
|
|
190
|
-
elif "datetime" in dataset_item:
|
|
191
|
-
properties["startTimeFromAscendingNode"] = dataset_item["datetime"]
|
|
192
|
-
properties["completionTimeFromAscendingNode"] = dataset_item["datetime"]
|
|
216
|
+
dates = _get_dates_from_dataset_data(dataset_item)
|
|
217
|
+
if not dates:
|
|
218
|
+
return None
|
|
219
|
+
properties["startTimeFromAscendingNode"] = dates["start"]
|
|
220
|
+
properties["completionTimeFromAscendingNode"] = dates["end"]
|
|
193
221
|
else:
|
|
194
|
-
item_dates = re.findall(r"\d{
|
|
222
|
+
item_dates = re.findall(r"(\d{4})(0[1-9]|1[0-2])([0-3]\d)", item_id)
|
|
195
223
|
if not item_dates:
|
|
196
|
-
item_dates = re.findall(r"\d{
|
|
224
|
+
item_dates = re.findall(r"_(\d{4})(0[1-9]|1[0-2])", item_id)
|
|
225
|
+
item_dates = ["".join(row) for row in item_dates]
|
|
197
226
|
item_start = _get_date_from_yyyymmdd(item_dates[0], item_key)
|
|
198
227
|
if not item_start: # identified pattern was not a valid datetime
|
|
199
228
|
return None
|
|
@@ -209,11 +238,26 @@ class CopMarineSearch(StaticStacSearch):
|
|
|
209
238
|
).strftime("%Y-%m-%dT%H:%M:%SZ")
|
|
210
239
|
|
|
211
240
|
for key, value in collection_dict["properties"].items():
|
|
212
|
-
if key not in ["id", "title", "start_datetime", "end_datetime"]:
|
|
241
|
+
if key not in ["id", "title", "start_datetime", "end_datetime", "datetime"]:
|
|
213
242
|
properties[key] = value
|
|
214
243
|
for key, value in dataset_item["properties"].items():
|
|
215
|
-
if key not in ["id", "title", "start_datetime", "end_datetime"]:
|
|
244
|
+
if key not in ["id", "title", "start_datetime", "end_datetime", "datetime"]:
|
|
216
245
|
properties[key] = value
|
|
246
|
+
|
|
247
|
+
code_mapping = self.config.products.get(product_type, {}).get(
|
|
248
|
+
"code_mapping", None
|
|
249
|
+
)
|
|
250
|
+
if code_mapping:
|
|
251
|
+
id_parts = item_id.split("_")
|
|
252
|
+
if len(id_parts) > code_mapping["index"]:
|
|
253
|
+
code = id_parts[code_mapping["index"]]
|
|
254
|
+
if "pattern" not in code_mapping:
|
|
255
|
+
properties[code_mapping["param"]] = code
|
|
256
|
+
elif re.findall(code_mapping["pattern"], code):
|
|
257
|
+
properties[code_mapping["param"]] = re.findall(
|
|
258
|
+
code_mapping["pattern"], code
|
|
259
|
+
)[0]
|
|
260
|
+
|
|
217
261
|
_check_int_values_properties(properties)
|
|
218
262
|
|
|
219
263
|
properties["thumbnail"] = collection_dict["assets"]["thumbnail"]["href"]
|
|
@@ -348,16 +392,54 @@ class CopMarineSearch(StaticStacSearch):
|
|
|
348
392
|
|
|
349
393
|
for obj in s3_objects["Contents"]:
|
|
350
394
|
item_key = obj["Key"]
|
|
395
|
+
item_id = os.path.splitext(item_key.split("/")[-1])[0]
|
|
351
396
|
# filter according to date(s) in item id
|
|
352
|
-
item_dates = re.findall(r"\d{
|
|
397
|
+
item_dates = re.findall(r"(\d{4})(0[1-9]|1[0-2])([0-3]\d)", item_id)
|
|
353
398
|
if not item_dates:
|
|
354
|
-
item_dates = re.findall(r"\d{
|
|
355
|
-
|
|
356
|
-
|
|
399
|
+
item_dates = re.findall(r"_(\d{4})(0[1-9]|1[0-2])", item_id)
|
|
400
|
+
item_dates = [
|
|
401
|
+
"".join(row) for row in item_dates
|
|
402
|
+
] # join tuples returned by findall
|
|
403
|
+
item_start = None
|
|
404
|
+
item_end = None
|
|
405
|
+
use_dataset_dates = False
|
|
406
|
+
if item_dates:
|
|
407
|
+
item_start = _get_date_from_yyyymmdd(item_dates[0], item_key)
|
|
408
|
+
if len(item_dates) > 2: # start, end and created_at timestamps
|
|
409
|
+
item_end = _get_date_from_yyyymmdd(item_dates[1], item_key)
|
|
410
|
+
if not item_start:
|
|
411
|
+
# no valid datetime given in id
|
|
412
|
+
use_dataset_dates = True
|
|
413
|
+
dates = _get_dates_from_dataset_data(dataset_item)
|
|
414
|
+
if dates:
|
|
415
|
+
item_start_str = dates["start"].replace("Z", "+0000")
|
|
416
|
+
item_end_str = dates["end"].replace("Z", "+0000")
|
|
417
|
+
try:
|
|
418
|
+
item_start = datetime.strptime(
|
|
419
|
+
item_start_str, "%Y-%m-%dT%H:%M:%S.%f%z"
|
|
420
|
+
)
|
|
421
|
+
item_end = datetime.strptime(
|
|
422
|
+
item_end_str, "%Y-%m-%dT%H:%M:%S.%f%z"
|
|
423
|
+
)
|
|
424
|
+
except ValueError:
|
|
425
|
+
item_start = datetime.strptime(
|
|
426
|
+
item_start_str, "%Y-%m-%dT%H:%M:%S%z"
|
|
427
|
+
)
|
|
428
|
+
item_end = datetime.strptime(
|
|
429
|
+
item_end_str, "%Y-%m-%dT%H:%M:%S%z"
|
|
430
|
+
)
|
|
431
|
+
if not item_start:
|
|
432
|
+
# no valid datetime in id and dataset data
|
|
357
433
|
continue
|
|
358
434
|
if item_start > end_date:
|
|
359
435
|
stop_search = True
|
|
360
|
-
if
|
|
436
|
+
if (
|
|
437
|
+
(start_date <= item_start <= end_date)
|
|
438
|
+
or (item_end and start_date <= item_end <= end_date)
|
|
439
|
+
or (
|
|
440
|
+
item_end and item_start < start_date and item_end > end_date
|
|
441
|
+
)
|
|
442
|
+
):
|
|
361
443
|
num_total += 1
|
|
362
444
|
if num_total < start_index:
|
|
363
445
|
continue
|
|
@@ -368,6 +450,7 @@ class CopMarineSearch(StaticStacSearch):
|
|
|
368
450
|
endpoint_url + "/" + bucket,
|
|
369
451
|
dataset_item,
|
|
370
452
|
collection_dict,
|
|
453
|
+
use_dataset_dates,
|
|
371
454
|
)
|
|
372
455
|
if product:
|
|
373
456
|
products.append(product)
|
|
@@ -29,7 +29,12 @@ from eodag.config import PluginConfig
|
|
|
29
29
|
from eodag.plugins.authentication.aws_auth import AwsAuth
|
|
30
30
|
from eodag.plugins.search.qssearch import ODataV4Search
|
|
31
31
|
from eodag.utils import guess_file_type
|
|
32
|
-
from eodag.utils.exceptions import
|
|
32
|
+
from eodag.utils.exceptions import (
|
|
33
|
+
AuthenticationError,
|
|
34
|
+
MisconfiguredError,
|
|
35
|
+
NotAvailableError,
|
|
36
|
+
RequestError,
|
|
37
|
+
)
|
|
33
38
|
|
|
34
39
|
DATA_EXTENSIONS = ["jp2", "tiff", "nc", "grib"]
|
|
35
40
|
logger = logging.getLogger("eodag.search.creodiass3")
|
|
@@ -50,7 +55,7 @@ def patched_register_downloader(self, downloader, authenticator):
|
|
|
50
55
|
try:
|
|
51
56
|
_update_assets(self, downloader.config, authenticator)
|
|
52
57
|
except BotoCoreError as e:
|
|
53
|
-
raise RequestError(
|
|
58
|
+
raise RequestError.from_error(e, "could not update assets") from e
|
|
54
59
|
|
|
55
60
|
|
|
56
61
|
def _update_assets(product: EOProduct, config: PluginConfig, auth: AwsAuth):
|
|
@@ -70,7 +75,7 @@ def _update_assets(product: EOProduct, config: PluginConfig, auth: AwsAuth):
|
|
|
70
75
|
if not getattr(auth, "s3_client", None):
|
|
71
76
|
auth.s3_client = boto3.client(
|
|
72
77
|
"s3",
|
|
73
|
-
endpoint_url=config.
|
|
78
|
+
endpoint_url=config.s3_endpoint,
|
|
74
79
|
aws_access_key_id=auth_dict["aws_access_key_id"],
|
|
75
80
|
aws_secret_access_key=auth_dict["aws_secret_access_key"],
|
|
76
81
|
)
|
|
@@ -105,12 +110,22 @@ def _update_assets(product: EOProduct, config: PluginConfig, auth: AwsAuth):
|
|
|
105
110
|
raise AuthenticationError(
|
|
106
111
|
f"Authentication failed on {config.base_uri} s3"
|
|
107
112
|
) from e
|
|
108
|
-
raise
|
|
113
|
+
raise NotAvailableError(
|
|
114
|
+
f"assets for product {prefix} could not be found"
|
|
115
|
+
) from e
|
|
109
116
|
|
|
110
117
|
|
|
111
118
|
class CreodiasS3Search(ODataV4Search):
|
|
112
119
|
"""
|
|
113
|
-
|
|
120
|
+
``CreodiasS3Search`` is an extension of :class:`~eodag.plugins.search.qssearch.ODataV4Search`,
|
|
121
|
+
it executes a Search on creodias and adapts results so that the assets contain links to s3.
|
|
122
|
+
It has the same configuration parameters as :class:`~eodag.plugins.search.qssearch.ODataV4Search` and
|
|
123
|
+
one additional parameter:
|
|
124
|
+
|
|
125
|
+
:param provider: provider name
|
|
126
|
+
:param config: Search plugin configuration:
|
|
127
|
+
|
|
128
|
+
* :attr:`~eodag.config.PluginConfig.s3_endpoint` (``str``) (**mandatory**): base url of the s3
|
|
114
129
|
"""
|
|
115
130
|
|
|
116
131
|
def __init__(self, provider, config):
|
eodag/plugins/search/csw.py
CHANGED
|
@@ -52,7 +52,47 @@ SUPPORTED_REFERENCE_SCHEMES = ["WWW:DOWNLOAD-1.0-http--download"]
|
|
|
52
52
|
|
|
53
53
|
|
|
54
54
|
class CSWSearch(Search):
|
|
55
|
-
"""A plugin for implementing search based on OGC CSW
|
|
55
|
+
"""A plugin for implementing search based on OGC CSW
|
|
56
|
+
|
|
57
|
+
:param provider: provider name
|
|
58
|
+
:param config: Search plugin configuration:
|
|
59
|
+
|
|
60
|
+
* :attr:`~eodag.config.PluginConfig.api_endpoint` (``str``) (**mandatory**): The endpoint of the
|
|
61
|
+
provider's search interface
|
|
62
|
+
* :attr:`~eodag.config.PluginConfig.version` (``str``): OGC Catalogue Service version; default: ``2.0.2``
|
|
63
|
+
* :attr:`~eodag.config.PluginConfig.search_definition` (``Dict[str, Any]``) (**mandatory**):
|
|
64
|
+
|
|
65
|
+
* **product_type_tags** (``List[Dict[str, Any]``): dict of product type tags
|
|
66
|
+
* **resource_location_filter** (``str``): regex string
|
|
67
|
+
* **date_tags** (``Dict[str, Any]``): tags for start and end
|
|
68
|
+
|
|
69
|
+
* :attr:`~eodag.config.PluginConfig.metadata_mapping` (``Dict[str, Any]``): The search plugins of this kind can
|
|
70
|
+
detect when a metadata mapping is "query-able", and get the semantics of how to format the query string
|
|
71
|
+
parameter that enables to make a query on the corresponding metadata. To make a metadata query-able,
|
|
72
|
+
just configure it in the metadata mapping to be a list of 2 items, the first one being the
|
|
73
|
+
specification of the query string search formatting. The later is a string following the
|
|
74
|
+
specification of Python string formatting, with a special behaviour added to it. For example,
|
|
75
|
+
an entry in the metadata mapping of this kind::
|
|
76
|
+
|
|
77
|
+
completionTimeFromAscendingNode:
|
|
78
|
+
- 'f=acquisition.endViewingDate:lte:{completionTimeFromAscendingNode#timestamp}'
|
|
79
|
+
- '$.properties.acquisition.endViewingDate'
|
|
80
|
+
|
|
81
|
+
means that the search url will have a query string parameter named ``f`` with a value of
|
|
82
|
+
``acquisition.endViewingDate:lte:1543922280.0`` if the search was done with the value
|
|
83
|
+
of ``completionTimeFromAscendingNode`` being ``2018-12-04T12:18:00``. What happened is that
|
|
84
|
+
``{completionTimeFromAscendingNode#timestamp}`` was replaced with the timestamp of the value
|
|
85
|
+
of ``completionTimeFromAscendingNode``. This example shows all there is to know about the
|
|
86
|
+
semantics of the query string formatting introduced by this plugin: any eodag search parameter
|
|
87
|
+
can be referenced in the query string with an additional optional conversion function that
|
|
88
|
+
is separated from it by a ``#`` (see :func:`~eodag.api.product.metadata_mapping.format_metadata` for further
|
|
89
|
+
details on the available converters). Note that for the values in the
|
|
90
|
+
:attr:`~eodag.config.PluginConfig.free_text_search_operations` configuration parameter follow the same rule.
|
|
91
|
+
If the metadata_mapping is not a list but only a string, this means that the parameters is not queryable but
|
|
92
|
+
it is included in the result obtained from the provider. The string indicates how the provider result should
|
|
93
|
+
be mapped to the eodag parameter.
|
|
94
|
+
|
|
95
|
+
"""
|
|
56
96
|
|
|
57
97
|
def __init__(self, provider: str, config: PluginConfig) -> None:
|
|
58
98
|
super(CSWSearch, self).__init__(provider, config)
|