eodag 3.0.0b3__py3-none-any.whl → 3.0.1__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (71) hide show
  1. eodag/api/core.py +189 -125
  2. eodag/api/product/metadata_mapping.py +12 -3
  3. eodag/api/search_result.py +29 -3
  4. eodag/cli.py +35 -19
  5. eodag/config.py +412 -116
  6. eodag/plugins/apis/base.py +10 -4
  7. eodag/plugins/apis/ecmwf.py +14 -4
  8. eodag/plugins/apis/usgs.py +25 -2
  9. eodag/plugins/authentication/aws_auth.py +14 -5
  10. eodag/plugins/authentication/base.py +10 -1
  11. eodag/plugins/authentication/generic.py +14 -3
  12. eodag/plugins/authentication/header.py +12 -4
  13. eodag/plugins/authentication/keycloak.py +41 -22
  14. eodag/plugins/authentication/oauth.py +11 -1
  15. eodag/plugins/authentication/openid_connect.py +178 -163
  16. eodag/plugins/authentication/qsauth.py +12 -4
  17. eodag/plugins/authentication/sas_auth.py +19 -2
  18. eodag/plugins/authentication/token.py +57 -10
  19. eodag/plugins/authentication/token_exchange.py +19 -19
  20. eodag/plugins/crunch/base.py +4 -1
  21. eodag/plugins/crunch/filter_date.py +5 -2
  22. eodag/plugins/crunch/filter_latest_intersect.py +5 -4
  23. eodag/plugins/crunch/filter_latest_tpl_name.py +1 -1
  24. eodag/plugins/crunch/filter_overlap.py +5 -7
  25. eodag/plugins/crunch/filter_property.py +4 -3
  26. eodag/plugins/download/aws.py +39 -22
  27. eodag/plugins/download/base.py +11 -11
  28. eodag/plugins/download/creodias_s3.py +11 -2
  29. eodag/plugins/download/http.py +86 -52
  30. eodag/plugins/download/s3rest.py +20 -18
  31. eodag/plugins/manager.py +168 -23
  32. eodag/plugins/search/base.py +33 -14
  33. eodag/plugins/search/build_search_result.py +55 -51
  34. eodag/plugins/search/cop_marine.py +112 -29
  35. eodag/plugins/search/creodias_s3.py +20 -5
  36. eodag/plugins/search/csw.py +41 -1
  37. eodag/plugins/search/data_request_search.py +109 -9
  38. eodag/plugins/search/qssearch.py +532 -152
  39. eodag/plugins/search/static_stac_search.py +20 -21
  40. eodag/resources/ext_product_types.json +1 -1
  41. eodag/resources/product_types.yml +187 -56
  42. eodag/resources/providers.yml +1610 -1701
  43. eodag/resources/stac.yml +3 -163
  44. eodag/resources/user_conf_template.yml +112 -97
  45. eodag/rest/config.py +1 -2
  46. eodag/rest/constants.py +0 -1
  47. eodag/rest/core.py +61 -51
  48. eodag/rest/errors.py +181 -0
  49. eodag/rest/server.py +24 -325
  50. eodag/rest/stac.py +93 -544
  51. eodag/rest/types/eodag_search.py +13 -8
  52. eodag/rest/types/queryables.py +1 -2
  53. eodag/rest/types/stac_search.py +11 -2
  54. eodag/types/__init__.py +15 -3
  55. eodag/types/download_args.py +1 -1
  56. eodag/types/queryables.py +1 -2
  57. eodag/types/search_args.py +3 -3
  58. eodag/utils/__init__.py +77 -57
  59. eodag/utils/exceptions.py +23 -9
  60. eodag/utils/logging.py +37 -77
  61. eodag/utils/requests.py +1 -3
  62. eodag/utils/stac_reader.py +1 -1
  63. {eodag-3.0.0b3.dist-info → eodag-3.0.1.dist-info}/METADATA +11 -12
  64. eodag-3.0.1.dist-info/RECORD +109 -0
  65. {eodag-3.0.0b3.dist-info → eodag-3.0.1.dist-info}/WHEEL +1 -1
  66. {eodag-3.0.0b3.dist-info → eodag-3.0.1.dist-info}/entry_points.txt +1 -0
  67. eodag/resources/constraints/climate-dt.json +0 -13
  68. eodag/resources/constraints/extremes-dt.json +0 -8
  69. eodag-3.0.0b3.dist-info/RECORD +0 -110
  70. {eodag-3.0.0b3.dist-info → eodag-3.0.1.dist-info}/LICENSE +0 -0
  71. {eodag-3.0.0b3.dist-info → eodag-3.0.1.dist-info}/top_level.txt +0 -0
@@ -18,7 +18,7 @@
18
18
  from __future__ import annotations
19
19
 
20
20
  import logging
21
- from typing import TYPE_CHECKING
21
+ from typing import TYPE_CHECKING, Annotated, get_args
22
22
 
23
23
  import orjson
24
24
  from pydantic.fields import Field, FieldInfo
@@ -35,11 +35,9 @@ from eodag.types.queryables import Queryables
35
35
  from eodag.types.search_args import SortByList
36
36
  from eodag.utils import (
37
37
  GENERIC_PRODUCT_TYPE,
38
- Annotated,
39
38
  copy_deepcopy,
40
39
  deepcopy,
41
40
  format_dict_items,
42
- get_args,
43
41
  update_nested_dict,
44
42
  )
45
43
  from eodag.utils.exceptions import ValidationError
@@ -95,9 +93,9 @@ class Search(PluginTopic):
95
93
  ) -> Tuple[List[EOProduct], Optional[int]]:
96
94
  """Implementation of how the products must be searched goes here.
97
95
 
98
- This method must return a tuple with (1) a list of EOProduct instances (see eodag.api.product module)
99
- which will be processed by a Download plugin (2) and the total number of products matching
100
- the search criteria. If ``prep.count`` is False, the second element returned must be ``None``.
96
+ This method must return a tuple with (1) a list of :class:`~eodag.api.product._product.EOProduct` instances
97
+ which will be processed by a :class:`~eodag.plugins.download.base.Download` plugin (2) and the total number of
98
+ products matching the search criteria. If ``prep.count`` is False, the second element returned must be ``None``.
101
99
  """
102
100
  raise NotImplementedError("A Search plugin must implement a method named query")
103
101
 
@@ -108,9 +106,9 @@ class Search(PluginTopic):
108
106
  def discover_queryables(
109
107
  self, **kwargs: Any
110
108
  ) -> Optional[Dict[str, Annotated[Any, FieldInfo]]]:
111
- """Fetch queryables list from provider using `discover_queryables` conf
109
+ """Fetch queryables list from provider using :attr:`~eodag.config.PluginConfig.discover_queryables` conf
112
110
 
113
- :param kwargs: additional filters for queryables (`productType` and other search
111
+ :param kwargs: additional filters for queryables (``productType`` and other search
114
112
  arguments)
115
113
  :returns: fetched queryable parameters dict
116
114
  """
@@ -179,6 +177,27 @@ class Search(PluginTopic):
179
177
  else:
180
178
  return {}
181
179
 
180
+ def get_product_type_cfg_value(self, key: str, default: Any = None) -> Any:
181
+ """
182
+ Get the value of a configuration option specific to the current product type.
183
+
184
+ This method retrieves the value of a configuration option from the
185
+ ``product_type_config`` attribute. If the option is not found, the provided
186
+ default value is returned.
187
+
188
+ :param key: The configuration option key.
189
+ :type key: str
190
+ :param default: The default value to be returned if the option is not found (default is None).
191
+ :type default: Any
192
+
193
+ :return: The value of the specified configuration option or the default value.
194
+ :rtype: Any
195
+ """
196
+ product_type_cfg = getattr(self.config, "product_type_config", {})
197
+ non_none_cfg = {k: v for k, v in product_type_cfg.items() if v}
198
+
199
+ return non_none_cfg.get(key, default)
200
+
182
201
  def get_metadata_mapping(
183
202
  self, product_type: Optional[str] = None
184
203
  ) -> Dict[str, Union[str, List[str]]]:
@@ -194,10 +213,10 @@ class Search(PluginTopic):
194
213
  return self.config.metadata_mapping
195
214
 
196
215
  def get_sort_by_arg(self, kwargs: Dict[str, Any]) -> Optional[SortByList]:
197
- """Extract the "sort_by" argument from the kwargs or the provider default sort configuration
216
+ """Extract the ``sort_by`` argument from the kwargs or the provider default sort configuration
198
217
 
199
218
  :param kwargs: Search arguments
200
- :returns: The "sort_by" argument from the kwargs or the provider default sort configuration
219
+ :returns: The ``sort_by`` argument from the kwargs or the provider default sort configuration
201
220
  """
202
221
  # remove "sort_by" from search args if exists because it is not part of metadata mapping,
203
222
  # it will complete the query string or body once metadata mapping will be done
@@ -216,16 +235,16 @@ class Search(PluginTopic):
216
235
  self, sort_by_arg: SortByList
217
236
  ) -> Tuple[str, Dict[str, List[Dict[str, str]]]]:
218
237
  """Build the sorting part of the query string or body by transforming
219
- the "sort_by" argument into a provider-specific string or dictionnary
238
+ the ``sort_by`` argument into a provider-specific string or dictionary
220
239
 
221
- :param sort_by_arg: the "sort_by" argument in EODAG format
222
- :returns: The "sort_by" argument in provider-specific format
240
+ :param sort_by_arg: the ``sort_by`` argument in EODAG format
241
+ :returns: The ``sort_by`` argument in provider-specific format
223
242
  """
224
243
  if not hasattr(self.config, "sort"):
225
244
  raise ValidationError(f"{self.provider} does not support sorting feature")
226
245
  # TODO: remove this code block when search args model validation is embeded
227
246
  # remove duplicates
228
- sort_by_arg = list(set(sort_by_arg))
247
+ sort_by_arg = list(dict.fromkeys(sort_by_arg))
229
248
 
230
249
  sort_by_qs: str = ""
231
250
  sort_by_qp: Dict[str, Any] = {}
@@ -20,16 +20,27 @@ from __future__ import annotations
20
20
  import hashlib
21
21
  import logging
22
22
  from datetime import datetime, timedelta, timezone
23
- from typing import TYPE_CHECKING, Any, Dict, List, Optional, Set, Tuple, cast
23
+ from typing import (
24
+ TYPE_CHECKING,
25
+ Annotated,
26
+ Any,
27
+ Dict,
28
+ List,
29
+ Optional,
30
+ Set,
31
+ Tuple,
32
+ cast,
33
+ get_args,
34
+ )
24
35
  from urllib.parse import quote_plus, unquote_plus
25
36
 
26
37
  import geojson
27
38
  import orjson
28
39
  from dateutil.parser import isoparse
40
+ from dateutil.tz import tzutc
29
41
  from jsonpath_ng import Child, Fields, Root
30
42
  from pydantic import create_model
31
43
  from pydantic.fields import FieldInfo
32
- from typing_extensions import get_args
33
44
 
34
45
  from eodag.api.product import EOProduct
35
46
  from eodag.api.product.metadata_mapping import (
@@ -47,7 +58,6 @@ from eodag.types import json_field_definition_to_python, model_fields_to_annotat
47
58
  from eodag.types.queryables import CommonQueryables
48
59
  from eodag.utils import (
49
60
  DEFAULT_MISSION_START_DATE,
50
- Annotated,
51
61
  deepcopy,
52
62
  dict_items_recursive_sort,
53
63
  get_geometry_from_various,
@@ -71,21 +81,16 @@ class BuildPostSearchResult(PostJsonSearch):
71
81
  performs a POST request and uses its result to build a single :class:`~eodag.api.search_result.SearchResult`
72
82
  object.
73
83
 
74
- The available configuration parameters inherits from parent classes, with particularly
75
- for this plugin:
76
-
77
- - **api_endpoint**: (mandatory) The endpoint of the provider's search interface
84
+ The available configuration parameters are inherited from parent classes
85
+ (:class:`~eodag.plugins.search.qssearch.PostJsonSearch` and
86
+ :class:`~eodag.plugins.search.qssearch.QueryStringSearch`), with particularly for this plugin:
78
87
 
79
- - **pagination**: The configuration of how the pagination is done
80
- on the provider. It is a tree with the following nodes:
88
+ :param provider: provider name
89
+ :param config: Search plugin configuration:
81
90
 
82
- - *next_page_query_obj*: (optional) The additional parameters needed to perform
83
- search. These paramaters won't be included in result. This must be a json dict
84
- formatted like `{{"foo":"bar"}}` because it will be passed to a `.format()`
85
- method before being loaded as json.
91
+ * :attr:`~eodag.config.PluginConfig.remove_from_query` (``List[str]``): List of parameters
92
+ used to parse metadata but that must not be included to the query
86
93
 
87
- :param provider: An eodag providers configuration dictionary
88
- :param config: Path to the user configuration file
89
94
  """
90
95
 
91
96
  def count_hits(
@@ -110,8 +115,8 @@ class BuildPostSearchResult(PostJsonSearch):
110
115
  prep.url = prep.search_urls[0]
111
116
  prep.info_message = f"Sending search request: {prep.url}"
112
117
  prep.exception_message = (
113
- f"Skipping error while searching for {self.provider} "
114
- f"{self.__class__.__name__} instance:"
118
+ f"Skipping error while searching for {self.provider}"
119
+ f" {self.__class__.__name__} instance"
115
120
  )
116
121
  response = self._request(prep)
117
122
 
@@ -178,7 +183,7 @@ class BuildPostSearchResult(PostJsonSearch):
178
183
  result.update(results.product_type_def_params)
179
184
  result = dict(result, **{k: v for k, v in kwargs.items() if v is not None})
180
185
 
181
- # parse porperties
186
+ # parse properties
182
187
  parsed_properties = properties_from_json(
183
188
  result,
184
189
  self.config.metadata_mapping,
@@ -239,19 +244,18 @@ class BuildSearchResult(BuildPostSearchResult):
239
244
  This plugin builds a single :class:`~eodag.api.search_result.SearchResult` object
240
245
  using given query parameters as product properties.
241
246
 
242
- The available configuration parameters inherits from parent classes, with particularly
243
- for this plugin:
247
+ The available configuration parameters inherits from parent classes
248
+ (:class:`~eodag.plugins.search.build_search_result.BuildPostSearchResult`,
249
+ :class:`~eodag.plugins.search.qssearch.PostJsonSearch` and
250
+ :class:`~eodag.plugins.search.qssearch.QueryStringSearch`), with particularly for this plugin:
244
251
 
245
- - **end_date_excluded**: Set to `False` if provider does not include end date to
246
- search
252
+ :param provider: provider name
253
+ :param config: Search plugin configuration:
247
254
 
248
- - **remove_from_query**: List of parameters used to parse metadata but that must
249
- not be included to the query
255
+ * :attr:`~eodag.config.PluginConfig.end_date_excluded` (``bool``): Set to ``False`` if provider
256
+ does not include end date in the search request; In this case, if the end date is at midnight,
257
+ the previous day will be used. default: ``True``
250
258
 
251
- - **constraints_file_url**: url of the constraint file used to build queryables
252
-
253
- :param provider: An eodag providers configuration dictionary
254
- :param config: Path to the user configuration file
255
259
  """
256
260
 
257
261
  def __init__(self, provider: str, config: PluginConfig) -> None:
@@ -339,24 +343,6 @@ class BuildSearchResult(BuildPostSearchResult):
339
343
  self, product_type=product_type, **available_properties
340
344
  )
341
345
 
342
- def get_product_type_cfg(self, key: str, default: Any = None) -> Any:
343
- """
344
- Get the value of a configuration option specific to the current product type.
345
-
346
- This method retrieves the value of a configuration option from the
347
- `_product_type_config` attribute. If the option is not found, the provided
348
- default value is returned.
349
-
350
- :param key: The configuration option key.
351
- :param default: The default value to be returned if the option is not found (default is None).
352
-
353
- :return: The value of the specified configuration option or the default value.
354
- """
355
- product_type_cfg = getattr(self.config, "product_type_config", {})
356
- non_none_cfg = {k: v for k, v in product_type_cfg.items() if v}
357
-
358
- return non_none_cfg.get(key, default)
359
-
360
346
  def _preprocess_search_params(self, params: Dict[str, Any]) -> None:
361
347
  """Preprocess search parameters before making a request to the CDS API.
362
348
 
@@ -397,7 +383,7 @@ class BuildSearchResult(BuildPostSearchResult):
397
383
 
398
384
  # dates
399
385
  mission_start_dt = datetime.fromisoformat(
400
- self.get_product_type_cfg(
386
+ self.get_product_type_cfg_value(
401
387
  "missionStartDate", DEFAULT_MISSION_START_DATE
402
388
  ).replace(
403
389
  "Z", "+00:00"
@@ -423,11 +409,29 @@ class BuildSearchResult(BuildPostSearchResult):
423
409
  "completionTimeFromAscendingNode", default_end_str
424
410
  )
425
411
 
426
- # temporary _date parameter mixing start & end
412
+ # adapt end date if it is midnight
427
413
  end_date_excluded = getattr(self.config, "end_date_excluded", True)
428
- end_date = isoparse(params["completionTimeFromAscendingNode"])
429
- if not end_date_excluded and end_date == end_date.replace(
430
- hour=0, minute=0, second=0, microsecond=0
414
+ is_datetime = True
415
+ try:
416
+ end_date = datetime.strptime(
417
+ params["completionTimeFromAscendingNode"], "%Y-%m-%dT%H:%M:%SZ"
418
+ )
419
+ end_date = end_date.replace(tzinfo=tzutc())
420
+ except ValueError:
421
+ try:
422
+ end_date = datetime.strptime(
423
+ params["completionTimeFromAscendingNode"], "%Y-%m-%dT%H:%M:%S.%fZ"
424
+ )
425
+ end_date = end_date.replace(tzinfo=tzutc())
426
+ except ValueError:
427
+ end_date = isoparse(params["completionTimeFromAscendingNode"])
428
+ is_datetime = False
429
+ start_date = isoparse(params["startTimeFromAscendingNode"])
430
+ if (
431
+ not end_date_excluded
432
+ and is_datetime
433
+ and end_date > start_date
434
+ and end_date == end_date.replace(hour=0, minute=0, second=0, microsecond=0)
431
435
  ):
432
436
  end_date += timedelta(days=-1)
433
437
  params["completionTimeFromAscendingNode"] = end_date.isoformat()
@@ -19,6 +19,7 @@ from __future__ import annotations
19
19
 
20
20
  import copy
21
21
  import logging
22
+ import os
22
23
  import re
23
24
  from datetime import datetime
24
25
  from typing import TYPE_CHECKING, Any, Dict, List, Optional, Tuple, cast
@@ -37,7 +38,7 @@ from eodag.config import PluginConfig
37
38
  from eodag.plugins.search import PreparedSearch
38
39
  from eodag.plugins.search.static_stac_search import StaticStacSearch
39
40
  from eodag.utils import get_bucket_name_and_prefix
40
- from eodag.utils.exceptions import UnsupportedProductType, ValidationError
41
+ from eodag.utils.exceptions import RequestError, UnsupportedProductType, ValidationError
41
42
 
42
43
  if TYPE_CHECKING:
43
44
  from mypy_boto3_s3 import S3Client
@@ -67,6 +68,21 @@ def _get_date_from_yyyymmdd(date_str: str, item_key: str) -> Optional[datetime]:
67
68
  return date
68
69
 
69
70
 
71
+ def _get_dates_from_dataset_data(
72
+ dataset_item: Dict[str, Any]
73
+ ) -> Optional[Dict[str, str]]:
74
+ dates = {}
75
+ if "start_datetime" in dataset_item["properties"]:
76
+ dates["start"] = dataset_item["properties"]["start_datetime"]
77
+ dates["end"] = dataset_item["properties"]["end_datetime"]
78
+ elif "datetime" in dataset_item["properties"]:
79
+ dates["start"] = dataset_item["properties"]["datetime"]
80
+ dates["end"] = dataset_item["properties"]["datetime"]
81
+ else:
82
+ return None
83
+ return dates
84
+
85
+
70
86
  def _get_s3_client(endpoint_url: str) -> S3Client:
71
87
  s3_session = boto3.Session()
72
88
  return s3_session.client(
@@ -94,7 +110,21 @@ def _check_int_values_properties(properties: Dict[str, Any]):
94
110
 
95
111
 
96
112
  class CopMarineSearch(StaticStacSearch):
97
- """class that implements search for the Copernicus Marine provider"""
113
+ """class that implements search for the Copernicus Marine provider
114
+
115
+ It calls :meth:`~eodag.plugins.search.static_stac_search.StaticStacSearch.discover_product_types`
116
+ inherited from :class:`~eodag.plugins.search.static_stac_search.StaticStacSearch`
117
+ but for the actual search a special method which fetches the urls of the available products from an S3 storage and
118
+ filters them has been written.
119
+
120
+ The configuration parameters are inherited from the parent and grand-parent classes. The
121
+ :attr:`~eodag.config.PluginConfig.DiscoverMetadata.auto_discovery` parameter in the
122
+ :attr:`~eodag.config.PluginConfig.discover_metadata` section has to be set to ``false`` and the
123
+ :attr:`~eodag.config.PluginConfig.DiscoverQueryables.fetch_url` in the
124
+ :attr:`~eodag.config.PluginConfig.discover_queryables` queryables section has to be set to ``null`` to
125
+ overwrite the default config from the stac provider configuration because those functionalities
126
+ are not available.
127
+ """
98
128
 
99
129
  def __init__(self, provider: str, config: PluginConfig):
100
130
  original_metadata_mapping = copy.deepcopy(config.metadata_mapping)
@@ -107,12 +137,10 @@ class CopMarineSearch(StaticStacSearch):
107
137
  ) -> Tuple[Dict[str, Any], List[Dict[str, Any]]]:
108
138
  """Fetch product type and associated datasets info"""
109
139
 
110
- fetch_url = cast(
111
- str,
112
- self.config.discover_product_types["fetch_url"].format(
113
- **self.config.__dict__
114
- ),
140
+ fetch_url = cast(str, self.config.discover_product_types["fetch_url"]).format(
141
+ **self.config.__dict__
115
142
  )
143
+
116
144
  logger.debug("fetch data for collection %s", product_type)
117
145
  provider_product_type = self.config.products.get(product_type, {}).get(
118
146
  "productType", None
@@ -125,9 +153,14 @@ class CopMarineSearch(StaticStacSearch):
125
153
  )
126
154
  try:
127
155
  collection_data = requests.get(collection_url).json()
128
- except requests.RequestException:
156
+ except requests.RequestException as exc:
157
+ if exc.errno == 404:
158
+ logger.error("product %s not found", product_type)
159
+ raise UnsupportedProductType(product_type)
129
160
  logger.error("data for product %s could not be fetched", product_type)
130
- raise UnsupportedProductType(product_type)
161
+ raise RequestError.from_error(
162
+ exc, f"data for product {product_type} could not be fetched"
163
+ ) from exc
131
164
 
132
165
  datasets = []
133
166
  for link in [li for li in collection_data["links"] if li["rel"] == "item"]:
@@ -170,7 +203,7 @@ class CopMarineSearch(StaticStacSearch):
170
203
  use_dataset_dates: bool = False,
171
204
  ) -> Optional[EOProduct]:
172
205
 
173
- item_id = item_key.split("/")[-1].split(".")[0]
206
+ item_id = os.path.splitext(item_key.split("/")[-1])[0]
174
207
  download_url = s3_url + "/" + item_key
175
208
  properties = {
176
209
  "id": item_id,
@@ -180,20 +213,16 @@ class CopMarineSearch(StaticStacSearch):
180
213
  "dataset": dataset_item["id"],
181
214
  }
182
215
  if use_dataset_dates:
183
- if "start_datetime" in dataset_item:
184
- properties["startTimeFromAscendingNode"] = dataset_item[
185
- "start_datetime"
186
- ]
187
- properties["completionTimeFromAscendingNode"] = dataset_item[
188
- "end_datetime"
189
- ]
190
- elif "datetime" in dataset_item:
191
- properties["startTimeFromAscendingNode"] = dataset_item["datetime"]
192
- properties["completionTimeFromAscendingNode"] = dataset_item["datetime"]
216
+ dates = _get_dates_from_dataset_data(dataset_item)
217
+ if not dates:
218
+ return None
219
+ properties["startTimeFromAscendingNode"] = dates["start"]
220
+ properties["completionTimeFromAscendingNode"] = dates["end"]
193
221
  else:
194
- item_dates = re.findall(r"\d{8}", item_key)
222
+ item_dates = re.findall(r"(\d{4})(0[1-9]|1[0-2])([0-3]\d)", item_id)
195
223
  if not item_dates:
196
- item_dates = re.findall(r"\d{6}", item_key)
224
+ item_dates = re.findall(r"_(\d{4})(0[1-9]|1[0-2])", item_id)
225
+ item_dates = ["".join(row) for row in item_dates]
197
226
  item_start = _get_date_from_yyyymmdd(item_dates[0], item_key)
198
227
  if not item_start: # identified pattern was not a valid datetime
199
228
  return None
@@ -209,11 +238,26 @@ class CopMarineSearch(StaticStacSearch):
209
238
  ).strftime("%Y-%m-%dT%H:%M:%SZ")
210
239
 
211
240
  for key, value in collection_dict["properties"].items():
212
- if key not in ["id", "title", "start_datetime", "end_datetime"]:
241
+ if key not in ["id", "title", "start_datetime", "end_datetime", "datetime"]:
213
242
  properties[key] = value
214
243
  for key, value in dataset_item["properties"].items():
215
- if key not in ["id", "title", "start_datetime", "end_datetime"]:
244
+ if key not in ["id", "title", "start_datetime", "end_datetime", "datetime"]:
216
245
  properties[key] = value
246
+
247
+ code_mapping = self.config.products.get(product_type, {}).get(
248
+ "code_mapping", None
249
+ )
250
+ if code_mapping:
251
+ id_parts = item_id.split("_")
252
+ if len(id_parts) > code_mapping["index"]:
253
+ code = id_parts[code_mapping["index"]]
254
+ if "pattern" not in code_mapping:
255
+ properties[code_mapping["param"]] = code
256
+ elif re.findall(code_mapping["pattern"], code):
257
+ properties[code_mapping["param"]] = re.findall(
258
+ code_mapping["pattern"], code
259
+ )[0]
260
+
217
261
  _check_int_values_properties(properties)
218
262
 
219
263
  properties["thumbnail"] = collection_dict["assets"]["thumbnail"]["href"]
@@ -348,16 +392,54 @@ class CopMarineSearch(StaticStacSearch):
348
392
 
349
393
  for obj in s3_objects["Contents"]:
350
394
  item_key = obj["Key"]
395
+ item_id = os.path.splitext(item_key.split("/")[-1])[0]
351
396
  # filter according to date(s) in item id
352
- item_dates = re.findall(r"\d{8}", item_key)
397
+ item_dates = re.findall(r"(\d{4})(0[1-9]|1[0-2])([0-3]\d)", item_id)
353
398
  if not item_dates:
354
- item_dates = re.findall(r"\d{6}", item_key)
355
- item_start = _get_date_from_yyyymmdd(item_dates[0], item_key)
356
- if not item_start: # identified pattern was not a valid datetime
399
+ item_dates = re.findall(r"_(\d{4})(0[1-9]|1[0-2])", item_id)
400
+ item_dates = [
401
+ "".join(row) for row in item_dates
402
+ ] # join tuples returned by findall
403
+ item_start = None
404
+ item_end = None
405
+ use_dataset_dates = False
406
+ if item_dates:
407
+ item_start = _get_date_from_yyyymmdd(item_dates[0], item_key)
408
+ if len(item_dates) > 2: # start, end and created_at timestamps
409
+ item_end = _get_date_from_yyyymmdd(item_dates[1], item_key)
410
+ if not item_start:
411
+ # no valid datetime given in id
412
+ use_dataset_dates = True
413
+ dates = _get_dates_from_dataset_data(dataset_item)
414
+ if dates:
415
+ item_start_str = dates["start"].replace("Z", "+0000")
416
+ item_end_str = dates["end"].replace("Z", "+0000")
417
+ try:
418
+ item_start = datetime.strptime(
419
+ item_start_str, "%Y-%m-%dT%H:%M:%S.%f%z"
420
+ )
421
+ item_end = datetime.strptime(
422
+ item_end_str, "%Y-%m-%dT%H:%M:%S.%f%z"
423
+ )
424
+ except ValueError:
425
+ item_start = datetime.strptime(
426
+ item_start_str, "%Y-%m-%dT%H:%M:%S%z"
427
+ )
428
+ item_end = datetime.strptime(
429
+ item_end_str, "%Y-%m-%dT%H:%M:%S%z"
430
+ )
431
+ if not item_start:
432
+ # no valid datetime in id and dataset data
357
433
  continue
358
434
  if item_start > end_date:
359
435
  stop_search = True
360
- if not item_dates or (start_date <= item_start <= end_date):
436
+ if (
437
+ (start_date <= item_start <= end_date)
438
+ or (item_end and start_date <= item_end <= end_date)
439
+ or (
440
+ item_end and item_start < start_date and item_end > end_date
441
+ )
442
+ ):
361
443
  num_total += 1
362
444
  if num_total < start_index:
363
445
  continue
@@ -368,6 +450,7 @@ class CopMarineSearch(StaticStacSearch):
368
450
  endpoint_url + "/" + bucket,
369
451
  dataset_item,
370
452
  collection_dict,
453
+ use_dataset_dates,
371
454
  )
372
455
  if product:
373
456
  products.append(product)
@@ -29,7 +29,12 @@ from eodag.config import PluginConfig
29
29
  from eodag.plugins.authentication.aws_auth import AwsAuth
30
30
  from eodag.plugins.search.qssearch import ODataV4Search
31
31
  from eodag.utils import guess_file_type
32
- from eodag.utils.exceptions import AuthenticationError, MisconfiguredError, RequestError
32
+ from eodag.utils.exceptions import (
33
+ AuthenticationError,
34
+ MisconfiguredError,
35
+ NotAvailableError,
36
+ RequestError,
37
+ )
33
38
 
34
39
  DATA_EXTENSIONS = ["jp2", "tiff", "nc", "grib"]
35
40
  logger = logging.getLogger("eodag.search.creodiass3")
@@ -50,7 +55,7 @@ def patched_register_downloader(self, downloader, authenticator):
50
55
  try:
51
56
  _update_assets(self, downloader.config, authenticator)
52
57
  except BotoCoreError as e:
53
- raise RequestError(f"could not update assets: {str(e)}") from e
58
+ raise RequestError.from_error(e, "could not update assets") from e
54
59
 
55
60
 
56
61
  def _update_assets(product: EOProduct, config: PluginConfig, auth: AwsAuth):
@@ -70,7 +75,7 @@ def _update_assets(product: EOProduct, config: PluginConfig, auth: AwsAuth):
70
75
  if not getattr(auth, "s3_client", None):
71
76
  auth.s3_client = boto3.client(
72
77
  "s3",
73
- endpoint_url=config.base_uri,
78
+ endpoint_url=config.s3_endpoint,
74
79
  aws_access_key_id=auth_dict["aws_access_key_id"],
75
80
  aws_secret_access_key=auth_dict["aws_secret_access_key"],
76
81
  )
@@ -105,12 +110,22 @@ def _update_assets(product: EOProduct, config: PluginConfig, auth: AwsAuth):
105
110
  raise AuthenticationError(
106
111
  f"Authentication failed on {config.base_uri} s3"
107
112
  ) from e
108
- raise RequestError(f"assets for product {prefix} could not be found") from e
113
+ raise NotAvailableError(
114
+ f"assets for product {prefix} could not be found"
115
+ ) from e
109
116
 
110
117
 
111
118
  class CreodiasS3Search(ODataV4Search):
112
119
  """
113
- Search on creodias and adapt results to s3
120
+ ``CreodiasS3Search`` is an extension of :class:`~eodag.plugins.search.qssearch.ODataV4Search`,
121
+ it executes a Search on creodias and adapts results so that the assets contain links to s3.
122
+ It has the same configuration parameters as :class:`~eodag.plugins.search.qssearch.ODataV4Search` and
123
+ one additional parameter:
124
+
125
+ :param provider: provider name
126
+ :param config: Search plugin configuration:
127
+
128
+ * :attr:`~eodag.config.PluginConfig.s3_endpoint` (``str``) (**mandatory**): base url of the s3
114
129
  """
115
130
 
116
131
  def __init__(self, provider, config):
@@ -52,7 +52,47 @@ SUPPORTED_REFERENCE_SCHEMES = ["WWW:DOWNLOAD-1.0-http--download"]
52
52
 
53
53
 
54
54
  class CSWSearch(Search):
55
- """A plugin for implementing search based on OGC CSW"""
55
+ """A plugin for implementing search based on OGC CSW
56
+
57
+ :param provider: provider name
58
+ :param config: Search plugin configuration:
59
+
60
+ * :attr:`~eodag.config.PluginConfig.api_endpoint` (``str``) (**mandatory**): The endpoint of the
61
+ provider's search interface
62
+ * :attr:`~eodag.config.PluginConfig.version` (``str``): OGC Catalogue Service version; default: ``2.0.2``
63
+ * :attr:`~eodag.config.PluginConfig.search_definition` (``Dict[str, Any]``) (**mandatory**):
64
+
65
+ * **product_type_tags** (``List[Dict[str, Any]``): dict of product type tags
66
+ * **resource_location_filter** (``str``): regex string
67
+ * **date_tags** (``Dict[str, Any]``): tags for start and end
68
+
69
+ * :attr:`~eodag.config.PluginConfig.metadata_mapping` (``Dict[str, Any]``): The search plugins of this kind can
70
+ detect when a metadata mapping is "query-able", and get the semantics of how to format the query string
71
+ parameter that enables to make a query on the corresponding metadata. To make a metadata query-able,
72
+ just configure it in the metadata mapping to be a list of 2 items, the first one being the
73
+ specification of the query string search formatting. The later is a string following the
74
+ specification of Python string formatting, with a special behaviour added to it. For example,
75
+ an entry in the metadata mapping of this kind::
76
+
77
+ completionTimeFromAscendingNode:
78
+ - 'f=acquisition.endViewingDate:lte:{completionTimeFromAscendingNode#timestamp}'
79
+ - '$.properties.acquisition.endViewingDate'
80
+
81
+ means that the search url will have a query string parameter named ``f`` with a value of
82
+ ``acquisition.endViewingDate:lte:1543922280.0`` if the search was done with the value
83
+ of ``completionTimeFromAscendingNode`` being ``2018-12-04T12:18:00``. What happened is that
84
+ ``{completionTimeFromAscendingNode#timestamp}`` was replaced with the timestamp of the value
85
+ of ``completionTimeFromAscendingNode``. This example shows all there is to know about the
86
+ semantics of the query string formatting introduced by this plugin: any eodag search parameter
87
+ can be referenced in the query string with an additional optional conversion function that
88
+ is separated from it by a ``#`` (see :func:`~eodag.api.product.metadata_mapping.format_metadata` for further
89
+ details on the available converters). Note that for the values in the
90
+ :attr:`~eodag.config.PluginConfig.free_text_search_operations` configuration parameter follow the same rule.
91
+ If the metadata_mapping is not a list but only a string, this means that the parameters is not queryable but
92
+ it is included in the result obtained from the provider. The string indicates how the provider result should
93
+ be mapped to the eodag parameter.
94
+
95
+ """
56
96
 
57
97
  def __init__(self, provider: str, config: PluginConfig) -> None:
58
98
  super(CSWSearch, self).__init__(provider, config)