eodag 2.12.0__py3-none-any.whl → 3.0.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (93) hide show
  1. eodag/__init__.py +6 -8
  2. eodag/api/core.py +654 -538
  3. eodag/api/product/__init__.py +12 -2
  4. eodag/api/product/_assets.py +59 -16
  5. eodag/api/product/_product.py +100 -93
  6. eodag/api/product/drivers/__init__.py +7 -2
  7. eodag/api/product/drivers/base.py +0 -3
  8. eodag/api/product/metadata_mapping.py +192 -96
  9. eodag/api/search_result.py +69 -10
  10. eodag/cli.py +55 -25
  11. eodag/config.py +391 -116
  12. eodag/plugins/apis/base.py +11 -165
  13. eodag/plugins/apis/ecmwf.py +36 -25
  14. eodag/plugins/apis/usgs.py +80 -35
  15. eodag/plugins/authentication/aws_auth.py +13 -4
  16. eodag/plugins/authentication/base.py +10 -1
  17. eodag/plugins/authentication/generic.py +2 -2
  18. eodag/plugins/authentication/header.py +31 -6
  19. eodag/plugins/authentication/keycloak.py +17 -84
  20. eodag/plugins/authentication/oauth.py +3 -3
  21. eodag/plugins/authentication/openid_connect.py +268 -49
  22. eodag/plugins/authentication/qsauth.py +4 -1
  23. eodag/plugins/authentication/sas_auth.py +9 -2
  24. eodag/plugins/authentication/token.py +98 -47
  25. eodag/plugins/authentication/token_exchange.py +122 -0
  26. eodag/plugins/crunch/base.py +3 -1
  27. eodag/plugins/crunch/filter_date.py +3 -9
  28. eodag/plugins/crunch/filter_latest_intersect.py +0 -3
  29. eodag/plugins/crunch/filter_latest_tpl_name.py +1 -4
  30. eodag/plugins/crunch/filter_overlap.py +4 -8
  31. eodag/plugins/crunch/filter_property.py +5 -11
  32. eodag/plugins/download/aws.py +149 -185
  33. eodag/plugins/download/base.py +88 -97
  34. eodag/plugins/download/creodias_s3.py +1 -1
  35. eodag/plugins/download/http.py +638 -310
  36. eodag/plugins/download/s3rest.py +47 -45
  37. eodag/plugins/manager.py +228 -88
  38. eodag/plugins/search/__init__.py +36 -0
  39. eodag/plugins/search/base.py +239 -30
  40. eodag/plugins/search/build_search_result.py +382 -37
  41. eodag/plugins/search/cop_marine.py +441 -0
  42. eodag/plugins/search/creodias_s3.py +25 -20
  43. eodag/plugins/search/csw.py +5 -7
  44. eodag/plugins/search/data_request_search.py +61 -30
  45. eodag/plugins/search/qssearch.py +713 -255
  46. eodag/plugins/search/static_stac_search.py +106 -40
  47. eodag/resources/ext_product_types.json +1 -1
  48. eodag/resources/product_types.yml +1921 -34
  49. eodag/resources/providers.yml +4091 -3655
  50. eodag/resources/stac.yml +50 -216
  51. eodag/resources/stac_api.yml +71 -25
  52. eodag/resources/stac_provider.yml +5 -0
  53. eodag/resources/user_conf_template.yml +89 -32
  54. eodag/rest/__init__.py +6 -0
  55. eodag/rest/cache.py +70 -0
  56. eodag/rest/config.py +68 -0
  57. eodag/rest/constants.py +26 -0
  58. eodag/rest/core.py +735 -0
  59. eodag/rest/errors.py +178 -0
  60. eodag/rest/server.py +264 -431
  61. eodag/rest/stac.py +442 -836
  62. eodag/rest/types/collections_search.py +44 -0
  63. eodag/rest/types/eodag_search.py +238 -47
  64. eodag/rest/types/queryables.py +164 -0
  65. eodag/rest/types/stac_search.py +273 -0
  66. eodag/rest/utils/__init__.py +216 -0
  67. eodag/rest/utils/cql_evaluate.py +119 -0
  68. eodag/rest/utils/rfc3339.py +64 -0
  69. eodag/types/__init__.py +106 -10
  70. eodag/types/bbox.py +15 -14
  71. eodag/types/download_args.py +40 -0
  72. eodag/types/search_args.py +57 -7
  73. eodag/types/whoosh.py +79 -0
  74. eodag/utils/__init__.py +110 -91
  75. eodag/utils/constraints.py +37 -45
  76. eodag/utils/exceptions.py +39 -22
  77. eodag/utils/import_system.py +0 -4
  78. eodag/utils/logging.py +37 -80
  79. eodag/utils/notebook.py +4 -4
  80. eodag/utils/repr.py +113 -0
  81. eodag/utils/requests.py +128 -0
  82. eodag/utils/rest.py +100 -0
  83. eodag/utils/stac_reader.py +93 -21
  84. {eodag-2.12.0.dist-info → eodag-3.0.0.dist-info}/METADATA +88 -53
  85. eodag-3.0.0.dist-info/RECORD +109 -0
  86. {eodag-2.12.0.dist-info → eodag-3.0.0.dist-info}/WHEEL +1 -1
  87. {eodag-2.12.0.dist-info → eodag-3.0.0.dist-info}/entry_points.txt +7 -5
  88. eodag/plugins/apis/cds.py +0 -540
  89. eodag/rest/types/stac_queryables.py +0 -134
  90. eodag/rest/utils.py +0 -1133
  91. eodag-2.12.0.dist-info/RECORD +0 -94
  92. {eodag-2.12.0.dist-info → eodag-3.0.0.dist-info}/LICENSE +0 -0
  93. {eodag-2.12.0.dist-info → eodag-3.0.0.dist-info}/top_level.txt +0 -0
eodag/rest/stac.py CHANGED
@@ -21,37 +21,43 @@ import logging
21
21
  import os
22
22
  from collections import defaultdict
23
23
  from datetime import datetime, timezone
24
- from typing import TYPE_CHECKING, Any, Dict, List, Optional, Tuple, cast
25
- from urllib.parse import parse_qs, urlencode, urlparse
24
+ from typing import TYPE_CHECKING, Any, Dict, List, Optional
25
+ from urllib.parse import (
26
+ parse_qs,
27
+ quote,
28
+ urlencode,
29
+ urlparse,
30
+ urlsplit,
31
+ urlunparse,
32
+ urlunsplit,
33
+ )
26
34
 
27
- import dateutil.parser
28
35
  import geojson
29
- import shapefile
30
- from dateutil import tz
31
- from dateutil.relativedelta import relativedelta
32
- from shapely.geometry import shape
33
- from shapely.geometry.base import BaseGeometry
34
- from shapely.ops import unary_union
36
+ from jsonpath_ng.jsonpath import Child
35
37
 
36
38
  from eodag.api.product.metadata_mapping import (
37
39
  DEFAULT_METADATA_MAPPING,
38
40
  format_metadata,
39
41
  get_metadata_path,
40
42
  )
43
+ from eodag.rest.config import Settings
44
+ from eodag.rest.utils.rfc3339 import str_to_interval
41
45
  from eodag.utils import (
42
46
  deepcopy,
43
47
  dict_items_recursive_apply,
44
48
  format_dict_items,
49
+ guess_file_type,
45
50
  jsonpath_parse_dict_items,
46
51
  string_to_jsonpath,
47
52
  update_nested_dict,
48
- urljoin,
49
53
  )
50
54
  from eodag.utils.exceptions import (
51
55
  NoMatchingProductType,
52
56
  NotAvailableError,
53
- ValidationError,
57
+ RequestError,
58
+ TimeOutError,
54
59
  )
60
+ from eodag.utils.requests import fetch_json
55
61
 
56
62
  if TYPE_CHECKING:
57
63
  from eodag.api.core import EODataAccessGateway
@@ -61,23 +67,52 @@ if TYPE_CHECKING:
61
67
 
62
68
  logger = logging.getLogger("eodag.rest.stac")
63
69
 
64
- DEFAULT_MISSION_START_DATE = "2015-01-01T00:00:00Z"
65
- STAC_CATALOGS_PREFIX = "catalogs"
70
+ # fields not to put in item properties
71
+ COLLECTION_PROPERTIES = [
72
+ "abstract",
73
+ "instrument",
74
+ "platform",
75
+ "platformSerialIdentifier",
76
+ "processingLevel",
77
+ "sensorType",
78
+ "md5",
79
+ "license",
80
+ "title",
81
+ "missionStartDate",
82
+ "missionEndDate",
83
+ "keywords",
84
+ "stacCollection",
85
+ ]
86
+ IGNORED_ITEM_PROPERTIES = [
87
+ "_id",
88
+ "id",
89
+ "keyword",
90
+ "quicklook",
91
+ "thumbnail",
92
+ "downloadLink",
93
+ "orderLink",
94
+ "_dc_qs",
95
+ "qs",
96
+ "defaultGeometry",
97
+ "_date",
98
+ ]
99
+
100
+
101
+ def _quote_url_path(url: str) -> str:
102
+ parsed = urlsplit(url)
103
+ path = quote(parsed.path)
104
+ components = (parsed.scheme, parsed.netloc, path, parsed.query, parsed.fragment)
105
+ return urlunsplit(components)
66
106
 
67
107
 
68
108
  class StacCommon:
69
109
  """Stac common object
70
110
 
71
111
  :param url: Requested URL
72
- :type url: str
73
112
  :param stac_config: STAC configuration from stac.yml conf file
74
- :type stac_config: dict
75
113
  :param provider: (optional) Chosen provider
76
- :type provider: str
77
114
  :param eodag_api: EODAG python API instance
78
- :type eodag_api: :class:`eodag.api.core.EODataAccessGateway`
79
115
  :param root: (optional) API root
80
- :type root: str
81
116
  """
82
117
 
83
118
  def __init__(
@@ -100,7 +135,6 @@ class StacCommon:
100
135
  """Updates data using given input STAC dict data
101
136
 
102
137
  :param data: Catalog data (parsed STAC dict)
103
- :type data: dict
104
138
  """
105
139
  self.data.update(data)
106
140
 
@@ -112,15 +146,18 @@ class StacCommon:
112
146
  ):
113
147
  for i, bbox in enumerate(self.data["extent"]["spatial"]["bbox"]):
114
148
  self.data["extent"]["spatial"]["bbox"][i] = [float(x) for x in bbox]
115
- # "None" values to None
116
- self.data = dict_items_recursive_apply(
117
- self.data, lambda k, v: None if v == "None" else v
118
- )
119
149
 
120
- # ids and titles as str
121
- self.data = dict_items_recursive_apply(
122
- self.data, lambda k, v: str(v) if k in ["title", "id"] else v
123
- )
150
+ def apply_method_none(_: str, v: str) -> Optional[str]:
151
+ """ "None" values to None"""
152
+ return None if v == "None" else v
153
+
154
+ self.data = dict_items_recursive_apply(self.data, apply_method_none)
155
+
156
+ def apply_method_ids(k, v):
157
+ """ids and titles as str"""
158
+ return str(v) if k in ["title", "id"] else v
159
+
160
+ self.data = dict_items_recursive_apply(self.data, apply_method_ids)
124
161
 
125
162
  # empty stac_extensions: "" to []
126
163
  if not self.data.get("stac_extensions", True):
@@ -133,15 +170,10 @@ class StacCommon:
133
170
  """Parse STAC extension from config and return as dict
134
171
 
135
172
  :param url: Requested URL
136
- :type url: str
137
173
  :param stac_config: STAC configuration from stac.yml conf file
138
- :type stac_config: dict
139
174
  :param extension: Extension name
140
- :type extension: str
141
175
  :param kwargs: Additional variables needed for parsing extension
142
- :type kwargs: Any
143
- :returns: STAC extension as dictionnary
144
- :rtype: dict
176
+ :returns: STAC extension as dictionary
145
177
  """
146
178
  extension_model = deepcopy(stac_config).get("extensions", {}).get(extension, {})
147
179
 
@@ -153,30 +185,30 @@ class StacCommon:
153
185
  }
154
186
  return format_dict_items(extension_model, **format_args)
155
187
 
156
- def as_dict(self) -> Dict[str, Any]:
157
- """Returns object data as dictionnary
158
-
159
- :returns: STAC data dictionnary
160
- :rtype: dict
161
- """
162
- return geojson.loads(geojson.dumps(self.data))
163
-
164
- __geo_interface__ = property(as_dict)
188
+ def get_provider_dict(self, provider: str) -> Dict[str, Any]:
189
+ """Generate STAC provider dict"""
190
+ provider_config = next(
191
+ p
192
+ for p in self.eodag_api.providers_config.values()
193
+ if provider in [p.name, getattr(p, "group", None)]
194
+ )
195
+ return {
196
+ "name": getattr(provider_config, "group", provider_config.name),
197
+ "description": getattr(provider_config, "description", None),
198
+ "roles": getattr(provider_config, "roles", None),
199
+ "url": getattr(provider_config, "url", None),
200
+ "priority": getattr(provider_config, "priority", None),
201
+ }
165
202
 
166
203
 
167
204
  class StacItem(StacCommon):
168
205
  """Stac item object
169
206
 
170
207
  :param url: Requested URL
171
- :type url: str
172
208
  :param stac_config: STAC configuration from stac.yml conf file
173
- :type stac_config: dict
174
209
  :param provider: (optional) Chosen provider
175
- :type provider: str
176
210
  :param eodag_api: EODAG python API instance
177
- :type eodag_api: :class:`eodag.api.core.EODataAccessGateway`
178
211
  :param root: (optional) API root
179
- :type root: str
180
212
  """
181
213
 
182
214
  def __init__(
@@ -201,11 +233,8 @@ class StacItem(StacCommon):
201
233
  """Build STAC items list from EODAG search results
202
234
 
203
235
  :param search_results: EODAG search results
204
- :type search_results: :class:`~eodag.api.search_result.SearchResult`
205
236
  :param catalog: STAC catalog dict used for parsing item metadata
206
- :type catalog: dict
207
237
  :returns: STAC item dicts list
208
- :rtype: list
209
238
  """
210
239
  if len(search_results) <= 0:
211
240
  return []
@@ -213,7 +242,6 @@ class StacItem(StacCommon):
213
242
  item_model = self.__filter_item_model_properties(
214
243
  self.stac_config["item"], str(search_results[0].product_type)
215
244
  )
216
- provider_model = deepcopy(self.stac_config["provider"])
217
245
 
218
246
  # check if some items need to be converted
219
247
  need_conversion: Dict[str, Any] = {}
@@ -229,77 +257,62 @@ class StacItem(StacCommon):
229
257
  k, item_model["properties"][k]
230
258
  )
231
259
 
260
+ item_props = [
261
+ p.right.fields[0]
262
+ for p in item_model["properties"].values()
263
+ if isinstance(p, Child)
264
+ ]
265
+ ignored_props = COLLECTION_PROPERTIES + item_props + IGNORED_ITEM_PROPERTIES
266
+
232
267
  item_list: List[Dict[str, Any]] = []
233
268
  for product in search_results:
234
- # parse jsonpath
235
- provider_dict = jsonpath_parse_dict_items(
236
- provider_model,
237
- {
238
- "provider": self.eodag_api.providers_config[
239
- product.provider
240
- ].__dict__
241
- },
242
- )
243
-
244
269
  product_dict = deepcopy(product.__dict__)
245
- if isinstance(product.assets, dict):
246
- product_dict["assets"] = product.assets
247
- else:
248
- product_dict["assets"] = product.assets.as_dict()
249
270
 
250
- product_item = jsonpath_parse_dict_items(
271
+ product_item: Dict[str, Any] = jsonpath_parse_dict_items(
251
272
  item_model,
252
273
  {
253
274
  "product": product_dict,
254
- "providers": [provider_dict],
275
+ "providers": [self.get_provider_dict(product.provider)],
255
276
  },
256
277
  )
257
278
 
279
+ # add additional item props
280
+ for p in set(product.properties) - set(ignored_props):
281
+ prefix = getattr(
282
+ self.eodag_api.providers_config[product.provider],
283
+ "group",
284
+ product.provider,
285
+ )
286
+ key = p if ":" in p else f"{prefix}:{p}"
287
+ product_item["properties"][key] = product.properties[p]
288
+
258
289
  # parse download link
259
- url_parts = urlparse(str(product_item["assets"]["downloadLink"]["href"]))
290
+ downloadlink_href = (
291
+ f"{catalog['url']}/items/{product.properties['id']}/download"
292
+ )
293
+ _dc_qs = product.properties.get("_dc_qs")
294
+ url_parts = urlparse(downloadlink_href)
260
295
  query_dict = parse_qs(url_parts.query)
261
296
  without_arg_url = (
262
297
  f"{url_parts.scheme}://{url_parts.netloc}{url_parts.path}"
263
298
  if url_parts.scheme
264
299
  else f"{url_parts.netloc}{url_parts.path}"
265
300
  )
266
-
267
301
  # add provider to query-args
268
- if self.provider:
269
- query_dict.update(provider=[self.provider])
302
+ p_config = self.eodag_api.providers_config[product.provider]
303
+ query_dict.update(provider=[getattr(p_config, "group", p_config.name)])
270
304
  # add datacube query-string to query-args
271
- _dc_qs = product_item["assets"]["downloadLink"].pop("_dc_qs", None)
272
305
  if _dc_qs:
273
- query_dict.update(_dc_qs=_dc_qs)
274
-
275
- # update download link with up-to-date query-args
306
+ query_dict.update(_dc_qs=[_dc_qs])
276
307
  if query_dict:
277
- product_item["assets"]["downloadLink"][
278
- "href"
279
- ] = f"{without_arg_url}?{urlencode(query_dict, doseq=True)}"
308
+ downloadlink_href = (
309
+ f"{without_arg_url}?{urlencode(query_dict, doseq=True)}"
310
+ )
280
311
 
281
- # move origin asset urls to alternate links and replace with eodag-server ones
282
- origin_assets = product_item["assets"].pop("origin_assets", {})
283
- if getattr(product, "assets", False):
284
- # replace origin asset urls with eodag-server ones
285
- for asset_key, asset_value in origin_assets.items():
286
- # use origin asset as default
287
- product_item["assets"][asset_key] = asset_value
288
- # origin assets as alternate link
289
- product_item["assets"][asset_key]["alternate"] = {
290
- "origin": {
291
- "title": "Origin asset link",
292
- "href": asset_value["href"],
293
- }
294
- }
295
- # use server-mode assets download links
296
- asset_value["href"] = without_arg_url
297
- if query_dict:
298
- product_item["assets"][asset_key][
299
- "href"
300
- ] += f"/{asset_key}?{urlencode(query_dict, doseq=True)}"
301
- else:
302
- product_item["assets"][asset_key]["href"] += f"/{asset_key}"
312
+ # generate STAC assets
313
+ product_item["assets"] = self._get_assets(
314
+ product, downloadlink_href, without_arg_url, query_dict, _dc_qs
315
+ )
303
316
 
304
317
  # apply conversion if needed
305
318
  for prop_key, prop_val in need_conversion.items():
@@ -321,16 +334,23 @@ class StacItem(StacCommon):
321
334
  format_args = deepcopy(self.stac_config)
322
335
  format_args["catalog"] = catalog
323
336
  format_args["item"] = product_item
324
- product_item: Dict[str, Any] = format_dict_items(
325
- product_item, **format_args
326
- )
337
+ product_item = format_dict_items(product_item, **format_args)
327
338
  product_item["bbox"] = [float(i) for i in product_item["bbox"]]
328
339
 
340
+ # transform shapely geometry to geojson
341
+ product_item["geometry"] = geojson.loads(
342
+ geojson.dumps(product_item["geometry"])
343
+ )
344
+
329
345
  # remove empty properties
330
346
  product_item = self.__filter_item_properties_values(product_item)
331
347
 
348
+ # quote invalid characters in links
349
+ for link in product_item["links"]:
350
+ link["href"] = _quote_url_path(link["href"])
351
+
332
352
  # update item link with datacube query-string
333
- if _dc_qs:
353
+ if _dc_qs or self.provider:
334
354
  url_parts = urlparse(str(product_item["links"][0]["href"]))
335
355
  without_arg_url = (
336
356
  f"{url_parts.scheme}://{url_parts.netloc}{url_parts.path}"
@@ -345,50 +365,128 @@ class StacItem(StacCommon):
345
365
 
346
366
  return item_list
347
367
 
368
+ def _get_assets(
369
+ self,
370
+ product: EOProduct,
371
+ downloadlink_href: str,
372
+ without_arg_url: str,
373
+ query_dict: Optional[Dict[str, Any]] = None,
374
+ _dc_qs: Optional[str] = None,
375
+ ) -> Dict[str, Any]:
376
+ assets: Dict[str, Any] = {}
377
+ settings = Settings.from_environment()
378
+
379
+ if _dc_qs:
380
+ parsed = urlparse(product.remote_location)
381
+ fragments = parsed.fragment.split("?")
382
+ parsed = parsed._replace(fragment=f"{fragments[0]}?_dc_qs={_dc_qs}")
383
+ origin_href = urlunparse(parsed)
384
+ else:
385
+ origin_href = product.remote_location
386
+
387
+ # update download link with up-to-date query-args
388
+ quoted_href = _quote_url_path(
389
+ downloadlink_href
390
+ ) # quote invalid characters in url
391
+ assets["downloadLink"] = {
392
+ "title": "Download link",
393
+ "href": quoted_href,
394
+ "type": "application/zip",
395
+ }
396
+
397
+ if not origin_href.startswith(tuple(settings.origin_url_blacklist)):
398
+ assets["downloadLink"]["alternate"] = {
399
+ "origin": {
400
+ "title": "Origin asset link",
401
+ "href": origin_href,
402
+ }
403
+ }
404
+
405
+ if "storageStatus" in product.properties:
406
+ assets["downloadLink"]["storage:tier"] = product.properties["storageStatus"]
407
+
408
+ # move origin asset urls to alternate links and replace with eodag-server ones
409
+ if product.assets:
410
+ origin_assets = product.assets.as_dict()
411
+ # replace origin asset urls with eodag-server ones
412
+ for asset_key, asset_value in origin_assets.items():
413
+ # use origin asset as default
414
+ assets[asset_key] = asset_value
415
+ # origin assets as alternate link
416
+ if not asset_value["href"].startswith(
417
+ tuple(settings.origin_url_blacklist)
418
+ ):
419
+ assets[asset_key]["alternate"] = {
420
+ "origin": {
421
+ "title": "Origin asset link",
422
+ "href": asset_value["href"],
423
+ }
424
+ }
425
+ # use server-mode assets download links
426
+ asset_value["href"] = without_arg_url
427
+ if query_dict:
428
+ assets[asset_key][
429
+ "href"
430
+ ] += f"/{asset_key}?{urlencode(query_dict, doseq=True)}"
431
+ else:
432
+ assets[asset_key]["href"] += f"/{asset_key}"
433
+ if asset_type := asset_value.get("type", None):
434
+ assets[asset_key]["type"] = asset_type
435
+ if origin := assets[asset_key].get("alternate", {}).get("origin"):
436
+ origin["type"] = asset_type
437
+ asset_value["href"] = _quote_url_path(asset_value["href"])
438
+
439
+ if thumbnail_url := product.properties.get(
440
+ "quicklook", product.properties.get("thumbnail", None)
441
+ ):
442
+ assets["thumbnail"] = {
443
+ "title": "Thumbnail",
444
+ "href": thumbnail_url,
445
+ "role": "thumbnail",
446
+ }
447
+ if mime_type := guess_file_type(thumbnail_url):
448
+ assets["thumbnail"]["type"] = mime_type
449
+ return assets
450
+
348
451
  def get_stac_items(
349
- self, search_results: SearchResult, catalog: Dict[str, Any]
452
+ self,
453
+ search_results: SearchResult,
454
+ total: int,
455
+ catalog: Dict[str, Any],
456
+ next_link: Optional[Dict[str, Any]],
350
457
  ) -> Dict[str, Any]:
351
458
  """Build STAC items from EODAG search results
352
459
 
353
460
  :param search_results: EODAG search results
354
- :type search_results: :class:`~eodag.api.search_result.SearchResult`
355
461
  :param catalog: STAC catalog dict used for parsing item metadata
356
- :type catalog: dict
357
- :returns: Items dictionnary
358
- :rtype: dict
462
+ :returns: Items dictionary
359
463
  """
360
464
  items_model = deepcopy(self.stac_config["items"])
361
465
 
362
- search_results.numberMatched = search_results.properties["totalResults"]
363
- search_results.numberReturned = len(search_results)
364
-
365
- # next page link
366
466
  if "?" in self.url:
367
467
  # search endpoint: use page url as self link
368
468
  for i, _ in enumerate(items_model["links"]):
369
469
  if items_model["links"][i]["rel"] == "self":
370
470
  items_model["links"][i]["href"] = catalog["url"]
371
471
 
372
- search_results.timeStamp = (
373
- datetime.now(timezone.utc).isoformat().replace("+00:00", "") + "Z"
374
- )
472
+ timestamp = datetime.now(timezone.utc).isoformat().replace("+00:00", "Z")
375
473
 
376
474
  # parse jsonpath
377
475
  items = jsonpath_parse_dict_items(
378
- items_model, {"search_results": search_results.__dict__}
476
+ items_model,
477
+ {
478
+ "numberMatched": total,
479
+ "numberReturned": len(search_results),
480
+ "timeStamp": timestamp,
481
+ },
379
482
  )
380
483
  # parse f-strings
381
484
  format_args = deepcopy(self.stac_config)
382
485
  format_args["catalog"] = catalog
383
486
  items = format_dict_items(items, **format_args)
384
487
 
385
- # last page: remove next page link
386
- if (
387
- search_results.properties["itemsPerPage"]
388
- * search_results.properties["page"]
389
- >= search_results.properties["totalResults"]
390
- ):
391
- items["links"] = [link for link in items["links"] if link["rel"] != "next"]
488
+ if next_link:
489
+ items["links"].append(next_link)
392
490
 
393
491
  # provide static catalog to build features
394
492
  if "search?" in catalog["url"]:
@@ -402,7 +500,7 @@ class StacItem(StacCommon):
402
500
  items["features"] = self.__get_item_list(search_results, catalog)
403
501
 
404
502
  self.update_data(items)
405
- return geojson.loads(geojson.dumps(self.data))
503
+ return self.data
406
504
 
407
505
  def __filter_item_model_properties(
408
506
  self, item_model: Dict[str, Any], product_type: str
@@ -412,11 +510,8 @@ class StacItem(StacCommon):
412
510
  part of oseo extension.
413
511
 
414
512
  :param item_model: Item model from stac_config
415
- :type item_model: dict
416
513
  :param product_type: Product type
417
- :type product_type: str
418
514
  :returns: Filtered item model
419
- :rtype: dict
420
515
  """
421
516
  try:
422
517
  product_type_dict = [
@@ -427,12 +522,10 @@ class StacItem(StacCommon):
427
522
  if pt["ID"] == product_type
428
523
  or ("alias" in pt and pt["alias"] == product_type)
429
524
  ][0]
430
- except IndexError:
525
+ except IndexError as e:
431
526
  raise NoMatchingProductType(
432
- "Product type {} not available for {}".format(
433
- product_type, self.provider
434
- )
435
- )
527
+ f"Product type {product_type} not available for {self.provider}"
528
+ ) from e
436
529
 
437
530
  result_item_model = deepcopy(item_model)
438
531
  result_item_model["stac_extensions"] = list(
@@ -479,9 +572,7 @@ class StacItem(StacCommon):
479
572
  """Removes empty properties, unused extensions, and add missing extensions
480
573
 
481
574
  :param item: STAC item data
482
- :type item: dict
483
575
  :returns: Filtered item model
484
- :rtype: dict
485
576
  """
486
577
  all_extensions_dict: Dict[str, str] = deepcopy(
487
578
  self.stac_config["stac_extensions"]
@@ -512,21 +603,13 @@ class StacItem(StacCommon):
512
603
  """Build STAC item from EODAG product
513
604
 
514
605
  :param product: EODAG product
515
- :type product: :class:`eodag.api.product._product.EOProduct`
516
606
  :returns: STAC item
517
- :rtype: list
518
607
  """
519
608
  product_type = str(product.product_type)
520
609
 
521
610
  item_model = self.__filter_item_model_properties(
522
611
  self.stac_config["item"], product_type
523
612
  )
524
- provider_model = deepcopy(self.stac_config["provider"])
525
-
526
- provider_dict = jsonpath_parse_dict_items(
527
- provider_model,
528
- {"provider": self.eodag_api.providers_config[product.provider].__dict__},
529
- )
530
613
 
531
614
  catalog = StacCatalog(
532
615
  url=self.url.split("/items")[0],
@@ -534,7 +617,7 @@ class StacItem(StacCommon):
534
617
  root=self.root,
535
618
  provider=self.provider,
536
619
  eodag_api=self.eodag_api,
537
- catalogs=[product_type],
620
+ collection=product_type,
538
621
  )
539
622
 
540
623
  product_dict = deepcopy(product.__dict__)
@@ -545,14 +628,15 @@ class StacItem(StacCommon):
545
628
  item_model,
546
629
  {
547
630
  "product": product_dict,
548
- "providers": provider_dict,
631
+ "providers": [self.get_provider_dict(product.provider)],
549
632
  },
550
633
  )
551
634
  # parse f-strings
552
635
  format_args = deepcopy(self.stac_config)
553
- format_args["catalog"] = dict(
554
- catalog.as_dict(), **{"url": catalog.url, "root": catalog.root}
555
- )
636
+ format_args["catalog"] = {
637
+ **catalog.data,
638
+ **{"url": catalog.url, "root": catalog.root},
639
+ }
556
640
  format_args["item"] = product_item
557
641
  product_item = format_dict_items(product_item, **format_args)
558
642
  product_item["bbox"] = [float(i) for i in product_item["bbox"]]
@@ -561,24 +645,46 @@ class StacItem(StacCommon):
561
645
  product_item = self.__filter_item_properties_values(product_item)
562
646
 
563
647
  self.update_data(product_item)
564
- return self.as_dict()
648
+ return self.data
565
649
 
566
650
 
567
651
  class StacCollection(StacCommon):
568
652
  """Stac collection object
569
653
 
570
654
  :param url: Requested URL
571
- :type url: str
572
655
  :param stac_config: STAC configuration from stac.yml conf file
573
- :type stac_config: dict
574
656
  :param provider: (optional) Chosen provider
575
- :type provider: str
576
657
  :param eodag_api: EODAG python API instance
577
- :type eodag_api: :class:`eodag.api.core.EODataAccessGateway`
578
658
  :param root: (optional) API root
579
- :type root: str
580
659
  """
581
660
 
661
+ # External STAC collections
662
+ ext_stac_collections: Dict[str, Dict[str, Any]] = dict()
663
+
664
+ @classmethod
665
+ def fetch_external_stac_collections(cls, eodag_api: EODataAccessGateway) -> None:
666
+ """Load external STAC collections
667
+
668
+ :param eodag_api: EODAG python API instance
669
+ """
670
+ list_product_types = eodag_api.list_product_types(fetch_providers=False)
671
+ for product_type in list_product_types:
672
+ ext_stac_collection_path = product_type.get("stacCollection")
673
+ if not ext_stac_collection_path:
674
+ continue
675
+ logger.info(f"Fetching external STAC collection for {product_type['ID']}")
676
+
677
+ try:
678
+ ext_stac_collection = fetch_json(ext_stac_collection_path)
679
+ except (RequestError, TimeOutError) as e:
680
+ logger.debug(e)
681
+ logger.warning(
682
+ f"Could not read remote external STAC collection from {ext_stac_collection_path}",
683
+ )
684
+ ext_stac_collection = {}
685
+
686
+ cls.ext_stac_collections[product_type["ID"]] = ext_stac_collection
687
+
582
688
  def __init__(
583
689
  self,
584
690
  url: str,
@@ -595,165 +701,174 @@ class StacCollection(StacCommon):
595
701
  root=root,
596
702
  )
597
703
 
598
- def __get_product_types(
599
- self, filters: Optional[Dict[str, Any]] = None
600
- ) -> List[Dict[str, Any]]:
601
- """Returns a list of supported product types
704
+ def __list_product_type_providers(self, product_type: Dict[str, Any]) -> List[str]:
705
+ """Retrieve a list of providers for a given product type.
602
706
 
603
- :param filters: (optional) Additional filters for product types search
604
- :type filters: dict
605
- :returns: List of corresponding product types
606
- :rtype: list
707
+ :param product_type: Dictionary containing information about the product type.
708
+ :return: A list of provider names.
607
709
  """
608
- if filters is None:
609
- filters = {}
610
- try:
611
- guessed_product_types = self.eodag_api.guess_product_type(**filters)
612
- except NoMatchingProductType:
613
- guessed_product_types = []
614
- if guessed_product_types:
615
- product_types = [
616
- pt
617
- for pt in self.eodag_api.list_product_types(provider=self.provider)
618
- if pt["ID"] in guessed_product_types
619
- ]
620
- else:
621
- product_types = self.eodag_api.list_product_types(provider=self.provider)
622
- return product_types
710
+ if self.provider:
711
+ return [self.provider]
623
712
 
624
- def __get_collection_list(
625
- self, filters: Optional[Dict[str, Any]] = None
626
- ) -> List[Dict[str, Any]]:
627
- """Build STAC collections list
713
+ return [
714
+ plugin.provider
715
+ for plugin in self.eodag_api._plugins_manager.get_search_plugins(
716
+ product_type=product_type.get("_id", product_type["ID"])
717
+ )
718
+ ]
628
719
 
629
- :param filters: (optional) Additional filters for collections search
630
- :type filters: dict
631
- :returns: STAC collection dicts list
632
- :rtype: list
720
+ def __generate_stac_collection(
721
+ self, collection_model: Any, product_type: Dict[str, Any]
722
+ ) -> Dict[str, Any]:
723
+ """Generate a STAC collection dictionary for a given product type.
724
+
725
+ :param collection_model: The base model for the STAC collection.
726
+ :param product_type: Dictionary containing information about the product type.
727
+ :return: A dictionary representing the STAC collection for the given product type.
633
728
  """
634
- collection_model = deepcopy(self.stac_config["collection"])
635
- provider_model = deepcopy(self.stac_config["provider"])
729
+ providers = self.__list_product_type_providers(product_type)
636
730
 
637
- product_types = self.__get_product_types(filters)
731
+ providers_dict: Dict[str, Dict[str, Any]] = {}
732
+ for provider in providers:
733
+ p_dict = self.get_provider_dict(provider)
734
+ providers_dict.setdefault(p_dict["name"], p_dict)
735
+ providers_list = list(providers_dict.values())
638
736
 
639
- collection_list: List[Dict[str, Any]] = []
640
- for product_type in product_types:
641
- if self.provider:
642
- providers = [self.provider]
643
- else:
644
- # get available providers for each product_type
645
- providers = [
646
- plugin.provider
647
- for plugin in self.eodag_api._plugins_manager.get_search_plugins(
648
- product_type=(
649
- product_type.get("_id", None) or product_type["ID"]
650
- )
651
- )
652
- ]
653
- providers_models: List[Dict[str, Any]] = []
654
- for provider in providers:
655
- provider_m = jsonpath_parse_dict_items(
656
- provider_model,
657
- {"provider": self.eodag_api.providers_config[provider].__dict__},
658
- )
659
- providers_models.append(provider_m)
737
+ # parse jsonpath
738
+ product_type_collection = jsonpath_parse_dict_items(
739
+ collection_model,
740
+ {
741
+ "product_type": product_type,
742
+ "providers": providers_list,
743
+ },
744
+ )
745
+ # override EODAG's collection with the external collection
746
+ ext_stac_collection = deepcopy(
747
+ self.ext_stac_collections.get(product_type["ID"], {})
748
+ )
660
749
 
661
- # parse jsonpath
662
- product_type_collection = jsonpath_parse_dict_items(
663
- collection_model,
664
- {
665
- "product_type": product_type,
666
- "providers": providers_models,
667
- },
668
- )
669
- # parse f-strings
670
- format_args = deepcopy(self.stac_config)
671
- format_args["collection"] = dict(
672
- product_type_collection,
673
- **{"url": f"{self.url}/{product_type['ID']}", "root": self.root},
750
+ # update links (keep eodag links as defaults)
751
+ ext_stac_collection.setdefault("links", {})
752
+ for link in product_type_collection["links"]:
753
+ ext_stac_collection["links"] = [
754
+ x for x in ext_stac_collection["links"] if x["rel"] != link["rel"]
755
+ ]
756
+ ext_stac_collection["links"].append(link)
757
+
758
+ # merge "summaries"
759
+ ext_stac_collection["summaries"] = {
760
+ k: v
761
+ for k, v in {
762
+ **ext_stac_collection.get("summaries", {}),
763
+ **product_type_collection["summaries"],
764
+ }.items()
765
+ if v and any(v)
766
+ }
767
+
768
+ # merge "keywords" lists
769
+ try:
770
+ ext_stac_collection["keywords"] = [
771
+ k
772
+ for k in set(
773
+ ext_stac_collection.get("keywords", [])
774
+ + product_type_collection["keywords"]
775
+ )
776
+ if k is not None
777
+ ]
778
+ except TypeError as e:
779
+ logger.warning(
780
+ f"Could not merge keywords from external collection for {product_type['ID']}: {str(e)}"
674
781
  )
675
- product_type_collection = format_dict_items(
676
- product_type_collection, **format_args
782
+ logger.debug(
783
+ f"External collection keywords: {str(ext_stac_collection.get('keywords'))}, ",
784
+ f"Product type keywords: {str(product_type_collection['keywords'])}",
677
785
  )
678
786
 
679
- collection_list.append(product_type_collection)
787
+ # merge providers
788
+ if "providers" in ext_stac_collection:
789
+ ext_stac_collection["providers"] += product_type_collection["providers"]
680
790
 
681
- return collection_list
682
-
683
- def get_collections(
684
- self, filters: Optional[Dict[str, Any]] = None
685
- ) -> Dict[str, Any]:
686
- """Build STAC collections
687
-
688
- :param filters: (optional) Additional filters for collections search
689
- :type filters: dict
690
- :returns: Collections dictionnary
691
- :rtype: dict
692
- """
693
- collections = deepcopy(self.stac_config["collections"])
694
- collections["collections"] = self.__get_collection_list(filters)
791
+ product_type_collection.update(ext_stac_collection)
695
792
 
696
- # # parse f-strings
793
+ # parse f-strings
697
794
  format_args = deepcopy(self.stac_config)
698
- format_args["collections"].update({"url": self.url, "root": self.root})
795
+ format_args["collection"] = {
796
+ **product_type_collection,
797
+ **{
798
+ "url": self.url
799
+ if self.url.endswith(product_type["ID"])
800
+ else f"{self.url}/{product_type['ID']}",
801
+ "root": self.root,
802
+ },
803
+ }
804
+ product_type_collection = format_dict_items(
805
+ product_type_collection, **format_args
806
+ )
699
807
 
700
- collections["links"] = [
701
- format_dict_items(link, **format_args) for link in collections["links"]
702
- ]
808
+ return product_type_collection
703
809
 
704
- collections["links"] += [
705
- {
706
- "rel": "child",
707
- "title": collec["id"],
708
- "href": [
709
- link["href"] for link in collec["links"] if link["rel"] == "self"
710
- ][0],
711
- }
712
- for collec in collections["collections"]
713
- ]
810
+ def get_collection_list(
811
+ self,
812
+ collection: Optional[str] = None,
813
+ q: Optional[str] = None,
814
+ platform: Optional[str] = None,
815
+ instrument: Optional[str] = None,
816
+ constellation: Optional[str] = None,
817
+ datetime: Optional[str] = None,
818
+ ) -> List[Dict[str, Any]]:
819
+ """Build STAC collections list
714
820
 
715
- self.update_data(collections)
716
- return self.as_dict()
821
+ :param filters: (optional) Additional filters for collections search
822
+ :returns: STAC collection dicts list
823
+ """
824
+ collection_model = deepcopy(self.stac_config["collection"])
717
825
 
718
- def get_collection_by_id(self, collection_id: str) -> Dict[str, Any]:
719
- """Build STAC collection by its id
826
+ start, end = str_to_interval(datetime)
720
827
 
721
- :param collection_id: Product type as collection ID
722
- :type collection_id: str
723
- :returns: Collection dictionary
724
- :rtype: dict
725
- """
726
- collection_list = self.__get_collection_list(
727
- filters={"productType": collection_id}
828
+ all_pt = self.eodag_api.list_product_types(
829
+ provider=self.provider, fetch_providers=False
728
830
  )
729
831
 
730
- try:
731
- collection = collection_list[0]
732
- except IndexError:
733
- raise NotAvailableError("%s collection not found" % collection_id)
832
+ if any((collection, q, platform, instrument, constellation, datetime)):
833
+ try:
834
+ guessed_product_types = self.eodag_api.guess_product_type(
835
+ free_text=q,
836
+ platformSerialIdentifier=platform,
837
+ instrument=instrument,
838
+ platform=constellation,
839
+ productType=collection,
840
+ missionStartDate=start.isoformat() if start else None,
841
+ missionEndDate=end.isoformat() if end else None,
842
+ )
843
+ except NoMatchingProductType:
844
+ product_types = []
845
+ else:
846
+ product_types = [
847
+ pt for pt in all_pt if pt["ID"] in guessed_product_types
848
+ ]
849
+ else:
850
+ product_types = all_pt
734
851
 
735
- self.update_data(collection)
736
- return self.as_dict()
852
+ # list product types with all metadata using guessed ids
853
+ collection_list: List[Dict[str, Any]] = []
854
+ for product_type in product_types:
855
+ stac_collection = self.__generate_stac_collection(
856
+ collection_model, product_type
857
+ )
858
+ collection_list.append(stac_collection)
859
+
860
+ return collection_list
737
861
 
738
862
 
739
863
  class StacCatalog(StacCommon):
740
864
  """Stac Catalog object
741
865
 
742
866
  :param url: Requested URL
743
- :type url: str
744
867
  :param stac_config: STAC configuration from stac.yml conf file
745
- :type stac_config: dict
746
868
  :param provider: Chosen provider
747
- :type provider: (optional) str
748
869
  :param eodag_api: EODAG python API instance
749
- :type eodag_api: :class:`eodag.api.core.EODataAccessGateway`
750
870
  :param root: (optional) API root
751
- :type root: str
752
- :param catalogs: (optional) Catalogs list
753
- :type catalogs: list
754
- :param fetch_providers: (optional) Whether to fetch providers for new product
755
- types or not
756
- :type fetch_providers: bool
871
+ :param collection: (optional) product type id
757
872
  """
758
873
 
759
874
  def __init__(
@@ -763,8 +878,7 @@ class StacCatalog(StacCommon):
763
878
  provider: Optional[str],
764
879
  eodag_api: EODataAccessGateway,
765
880
  root: str = "/",
766
- catalogs: List[str] = [],
767
- fetch_providers: bool = True,
881
+ collection: Optional[str] = None,
768
882
  ) -> None:
769
883
  super(StacCatalog, self).__init__(
770
884
  url=url,
@@ -791,13 +905,12 @@ class StacCatalog(StacCommon):
791
905
  self.data["links"] += self.children
792
906
 
793
907
  # build catalog
794
- self.__build_stac_catalog(catalogs, fetch_providers=fetch_providers)
908
+ self.__build_stac_catalog(collection)
795
909
 
796
910
  def __update_data_from_catalog_config(self, catalog_config: Dict[str, Any]) -> bool:
797
911
  """Updates configuration and data using given input catalog config
798
912
 
799
913
  :param catalog_config: Catalog config, from yml stac_config[catalogs]
800
- :type catalog_config: dict
801
914
  """
802
915
  model = catalog_config["model"]
803
916
 
@@ -817,40 +930,66 @@ class StacCatalog(StacCommon):
817
930
 
818
931
  return True
819
932
 
820
- def set_children(self, children: List[Dict[str, Any]] = []) -> bool:
821
- """Set catalog children / links
933
+ def __build_stac_catalog(self, collection: Optional[str] = None) -> StacCatalog:
934
+ """Build nested catalog from catalag list
822
935
 
823
- :param children: (optional) Children list
824
- :type children: list
936
+ :param collection: (optional) product type id
937
+ :returns: This catalog obj
825
938
  """
826
- self.children = children
827
- self.data["links"] = [
828
- link for link in self.data["links"] if link["rel"] != "child"
829
- ]
830
- self.data["links"] += children
831
- return True
939
+ settings = Settings.from_environment()
940
+
941
+ if not collection:
942
+ # Build root catalog combined with landing page
943
+ self.__update_data_from_catalog_config(
944
+ {
945
+ "model": {
946
+ **deepcopy(self.stac_config["landing_page"]),
947
+ **{
948
+ "provider": self.provider,
949
+ "id": settings.stac_api_landing_id,
950
+ "title": settings.stac_api_title,
951
+ "description": settings.stac_api_description,
952
+ },
953
+ }
954
+ }
955
+ )
956
+ else:
957
+ self.set_stac_product_type_by_id(collection)
958
+ return self
832
959
 
833
960
  def set_stac_product_type_by_id(
834
- self, product_type: str, **kwargs: Any
961
+ self, product_type: str, **_: Any
835
962
  ) -> Dict[str, Any]:
836
963
  """Updates catalog with given product_type
837
964
 
838
965
  :param product_type: Product type
839
- :type product_type: str
840
966
  """
841
- collection = StacCollection(
967
+ collections = StacCollection(
842
968
  url=self.url,
843
969
  stac_config=self.stac_config,
844
970
  provider=self.provider,
845
971
  eodag_api=self.eodag_api,
846
972
  root=self.root,
847
- ).get_collection_by_id(product_type)
848
-
849
- cat_model = deepcopy(self.stac_config["catalogs"]["product_type"]["model"])
973
+ ).get_collection_list(collection=product_type)
974
+
975
+ if not collections:
976
+ raise NotAvailableError(f"Collection {product_type} does not exist.")
977
+
978
+ cat_model = {
979
+ "id": "{collection[id]}",
980
+ "title": "{collection[title]}",
981
+ "description": "{collection[description]}",
982
+ "extent": "{collection[extent]}",
983
+ "crs": "http://www.opengis.net/def/crs/OGC/1.3/CRS84",
984
+ "keywords": "{collection[keywords]}",
985
+ "license": "{collection[license]}",
986
+ "providers": "{collection[providers]}",
987
+ "summaries": "{collection[summaries]}",
988
+ }
850
989
  # parse f-strings
851
990
  format_args = deepcopy(self.stac_config)
852
991
  format_args["catalog"] = defaultdict(str, **self.data)
853
- format_args["collection"] = collection
992
+ format_args["collection"] = collections[0]
854
993
  try:
855
994
  parsed_dict: Dict[str, Any] = format_dict_items(cat_model, **format_args)
856
995
  except Exception:
@@ -860,539 +999,6 @@ class StacCatalog(StacCommon):
860
999
  self.update_data(parsed_dict)
861
1000
 
862
1001
  # update search args
863
- self.search_args.update({"product_type": product_type})
1002
+ self.search_args.update({"productType": product_type})
864
1003
 
865
1004
  return parsed_dict
866
-
867
- # get / set dates filters -------------------------------------------------
868
-
869
- def get_stac_years_list(self, **kwargs: Any) -> List[int]:
870
- """Get catalog available years list
871
-
872
- :returns: Years list
873
- :rtype: list
874
- """
875
- extent_date_min, extent_date_max = self.get_datetime_extent()
876
-
877
- return list(range(extent_date_min.year, extent_date_max.year + 1))
878
-
879
- def get_stac_months_list(self, **kwargs: Any) -> List[int]:
880
- """Get catalog available months list
881
-
882
- :returns: Months list
883
- :rtype: list
884
- """
885
- extent_date_min, extent_date_max = self.get_datetime_extent()
886
-
887
- return list(
888
- range(
889
- extent_date_min.month,
890
- (extent_date_max - relativedelta(days=1)).month + 1,
891
- )
892
- )
893
-
894
- def get_stac_days_list(self, **kwargs: Any) -> List[int]:
895
- """Get catalog available days list
896
-
897
- :returns: Days list
898
- :rtype: list
899
- """
900
- extent_date_min, extent_date_max = self.get_datetime_extent()
901
-
902
- return list(
903
- range(
904
- extent_date_min.day, (extent_date_max - relativedelta(days=1)).day + 1
905
- )
906
- )
907
-
908
- def set_stac_year_by_id(self, year: str, **kwargs: Any) -> Dict[str, Any]:
909
- """Updates and returns catalog with given year
910
-
911
- :param year: Year number
912
- :type year: str
913
- :returns: Updated catalog
914
- :rtype: dict
915
- """
916
- extent_date_min, extent_date_max = self.get_datetime_extent()
917
-
918
- datetime_min = max(
919
- [extent_date_min, dateutil.parser.parse("{}-01-01T00:00:00Z".format(year))]
920
- )
921
- datetime_max = min(
922
- [
923
- extent_date_max,
924
- dateutil.parser.parse("{}-01-01T00:00:00Z".format((year)))
925
- + relativedelta(years=1),
926
- ]
927
- )
928
-
929
- catalog_model = deepcopy(self.stac_config["catalogs"]["year"]["model"])
930
-
931
- parsed_dict = self.set_stac_date(datetime_min, datetime_max, catalog_model)
932
-
933
- return parsed_dict
934
-
935
- def set_stac_month_by_id(self, month: str, **kwargs: Any) -> Dict[str, Any]:
936
- """Updates and returns catalog with given month
937
-
938
- :param month: Month number
939
- :type month: str
940
- :returns: Updated catalog
941
- :rtype: dict
942
- """
943
- extent_date_min, extent_date_max = self.get_datetime_extent()
944
- year = extent_date_min.year
945
-
946
- datetime_min = max(
947
- [
948
- extent_date_min,
949
- dateutil.parser.parse("{}-{}-01T00:00:00Z".format(year, month)),
950
- ]
951
- )
952
- datetime_max = min(
953
- [
954
- extent_date_max,
955
- dateutil.parser.parse("{}-{}-01T00:00:00Z".format(year, month))
956
- + relativedelta(months=1),
957
- ]
958
- )
959
-
960
- catalog_model = deepcopy(self.stac_config["catalogs"]["month"]["model"])
961
-
962
- parsed_dict = self.set_stac_date(datetime_min, datetime_max, catalog_model)
963
-
964
- return parsed_dict
965
-
966
- def set_stac_day_by_id(self, day: str, **kwargs: Any) -> Dict[str, Any]:
967
- """Updates and returns catalog with given day
968
-
969
- :param day: Day number
970
- :type day: str
971
- :returns: Updated catalog
972
- :rtype: dict
973
- """
974
- extent_date_min, extent_date_max = self.get_datetime_extent()
975
- year = extent_date_min.year
976
- month = extent_date_min.month
977
-
978
- datetime_min = max(
979
- [
980
- extent_date_min,
981
- dateutil.parser.parse("{}-{}-{}T00:00:00Z".format(year, month, day)),
982
- ]
983
- )
984
- datetime_max = min(
985
- [
986
- extent_date_max,
987
- dateutil.parser.parse("{}-{}-{}T00:00:00Z".format(year, month, day))
988
- + relativedelta(days=1),
989
- ]
990
- )
991
-
992
- catalog_model = deepcopy(self.stac_config["catalogs"]["day"]["model"])
993
-
994
- parsed_dict = self.set_stac_date(datetime_min, datetime_max, catalog_model)
995
-
996
- return parsed_dict
997
-
998
- def get_datetime_extent(self) -> Tuple[datetime, datetime]:
999
- """Returns catalog temporal extent as datetime objs
1000
-
1001
- :returns: Start & stop dates
1002
- :rtype: tuple
1003
- """
1004
- extent_date_min = dateutil.parser.parse(DEFAULT_MISSION_START_DATE).replace(
1005
- tzinfo=tz.UTC
1006
- )
1007
- extent_date_max = datetime.now(timezone.utc).replace(tzinfo=tz.UTC)
1008
- for interval in self.data["extent"]["temporal"]["interval"]:
1009
- extent_date_min_str, extent_date_max_str = interval
1010
- # date min
1011
- if extent_date_min_str:
1012
- extent_date_min = max(
1013
- extent_date_min, dateutil.parser.parse(extent_date_min_str)
1014
- )
1015
- # date max
1016
- if extent_date_max_str:
1017
- extent_date_max = min(
1018
- extent_date_max, dateutil.parser.parse(extent_date_max_str)
1019
- )
1020
-
1021
- return (
1022
- extent_date_min.replace(tzinfo=tz.UTC),
1023
- extent_date_max.replace(tzinfo=tz.UTC),
1024
- )
1025
-
1026
- def set_stac_date(
1027
- self,
1028
- datetime_min: datetime,
1029
- datetime_max: datetime,
1030
- catalog_model: Dict[str, Any],
1031
- ):
1032
- """Updates catalog data using given dates
1033
-
1034
- :param datetime_min: Date min of interval
1035
- :type datetime_min: :class:`datetime`
1036
- :param datetime_max: Date max of interval
1037
- :type datetime_max: :class:`datetime`
1038
- :param catalog_model: Catalog model to use, from yml stac_config[catalogs]
1039
- :type catalog_model: dict
1040
- :returns: Updated catalog
1041
- :rtype: dict
1042
- """
1043
- # parse f-strings
1044
- format_args = deepcopy(self.stac_config)
1045
- format_args["catalog"] = defaultdict(str, **self.data)
1046
- format_args["date"] = defaultdict(
1047
- str,
1048
- {
1049
- "year": datetime_min.year,
1050
- "month": datetime_min.month,
1051
- "day": datetime_min.day,
1052
- "min": datetime_min.isoformat().replace("+00:00", "") + "Z",
1053
- "max": datetime_max.isoformat().replace("+00:00", "") + "Z",
1054
- },
1055
- )
1056
- parsed_dict: Dict[str, Any] = format_dict_items(catalog_model, **format_args)
1057
-
1058
- self.update_data(parsed_dict)
1059
-
1060
- # update search args
1061
- self.search_args.update(
1062
- {
1063
- "dtstart": datetime_min.isoformat().split("T")[0],
1064
- "dtend": datetime_max.isoformat().split("T")[0],
1065
- }
1066
- )
1067
- return parsed_dict
1068
-
1069
- # get / set cloud_cover filter --------------------------------------------
1070
-
1071
- def get_stac_cloud_covers_list(self, **kwargs: Any) -> List[int]:
1072
- """Get cloud_cover list
1073
-
1074
- :returns: cloud_cover list
1075
- :rtype: list
1076
- """
1077
- return list(range(0, 101, 10))
1078
-
1079
- def set_stac_cloud_cover_by_id(
1080
- self, cloud_cover: str, **kwargs: Any
1081
- ) -> Dict[str, Any]:
1082
- """Updates and returns catalog with given max cloud_cover
1083
-
1084
- :param cloud_cover: Cloud_cover number
1085
- :type cloud_cover: str
1086
- :returns: Updated catalog
1087
- :rtype: dict
1088
- """
1089
- cat_model = deepcopy(self.stac_config["catalogs"]["cloud_cover"]["model"])
1090
- # parse f-strings
1091
- format_args = deepcopy(self.stac_config)
1092
- format_args["catalog"] = defaultdict(str, **self.data)
1093
- format_args["cloud_cover"] = cloud_cover
1094
- parsed_dict: Dict[str, Any] = format_dict_items(cat_model, **format_args)
1095
-
1096
- self.update_data(parsed_dict)
1097
-
1098
- # update search args
1099
- self.search_args.update({"query": {"eo:cloud_cover": {"lte": cloud_cover}}})
1100
-
1101
- return parsed_dict
1102
-
1103
- # get / set locations filter ----------------------------------------------
1104
-
1105
- def get_stac_location_list(self, catalog_name: str) -> List[str]:
1106
- """Get locations list using stac_conf & locations_config
1107
-
1108
- :param catalog_name: Catalog/location name
1109
- :type catalog_name: str
1110
- :returns: Locations list
1111
- :rtype: list
1112
- """
1113
-
1114
- if catalog_name not in self.stac_config["catalogs"]:
1115
- logger.warning(
1116
- "no entry found for {} in location_config".format(catalog_name)
1117
- )
1118
- return []
1119
- location_config = self.stac_config["catalogs"][catalog_name]
1120
-
1121
- for k in ["path", "attr"]:
1122
- if k not in location_config.keys():
1123
- logger.warning(
1124
- "no {} key found for {} in location_config".format(k, catalog_name)
1125
- )
1126
- return []
1127
- path = location_config["path"]
1128
- attr = location_config["attr"]
1129
-
1130
- with shapefile.Reader(path) as shp:
1131
- countries_list: List[str] = [rec[attr] for rec in shp.records()]
1132
-
1133
- # remove duplicates
1134
- countries_list = list(set(countries_list))
1135
-
1136
- countries_list.sort()
1137
-
1138
- return countries_list
1139
-
1140
- def set_stac_location_by_id(
1141
- self, location: str, catalog_name: str
1142
- ) -> Dict[str, Any]:
1143
- """Updates and returns catalog with given location
1144
-
1145
- :param location: Feature attribute value for shp filtering
1146
- :type location: str
1147
- :param catalog_name: Catalog/location name
1148
- :type catalog_name: str
1149
- :returns: Updated catalog
1150
- :rtype: dict
1151
- """
1152
- location_list_cat_key = catalog_name + "_list"
1153
-
1154
- if location_list_cat_key not in self.stac_config["catalogs"]:
1155
- logger.warning(
1156
- "no entry found for {}'s list in location_config".format(catalog_name)
1157
- )
1158
- return {}
1159
- location_config = self.stac_config["catalogs"][location_list_cat_key]
1160
-
1161
- for k in ["path", "attr"]:
1162
- if k not in location_config.keys():
1163
- logger.warning(
1164
- "no {} key found for {}'s list in location_config".format(
1165
- k, catalog_name
1166
- )
1167
- )
1168
- return {}
1169
- path = location_config["path"]
1170
- attr = location_config["attr"]
1171
-
1172
- with shapefile.Reader(path) as shp:
1173
- geom_hits = [
1174
- shape(shaperec.shape)
1175
- for shaperec in shp.shapeRecords()
1176
- if shaperec.record.as_dict().get(attr, None) == location
1177
- ]
1178
-
1179
- if len(geom_hits) == 0:
1180
- logger.warning(
1181
- "no feature found in %s matching %s=%s" % (path, attr, location)
1182
- )
1183
- return {}
1184
-
1185
- geom = cast(BaseGeometry, unary_union(geom_hits))
1186
-
1187
- cat_model = deepcopy(self.stac_config["catalogs"]["country"]["model"])
1188
- # parse f-strings
1189
- format_args = deepcopy(self.stac_config)
1190
- format_args["catalog"] = defaultdict(str, **self.data)
1191
- format_args["feature"] = defaultdict(str, {"geometry": geom, "id": location})
1192
- parsed_dict: Dict[str, Any] = format_dict_items(cat_model, **format_args)
1193
-
1194
- self.update_data(parsed_dict)
1195
-
1196
- # update search args
1197
- self.search_args.update({"geom": geom})
1198
-
1199
- return parsed_dict
1200
-
1201
- def build_locations_config(self) -> Dict[str, str]:
1202
- """Build locations config from stac_conf[locations_catalogs] & eodag_api.locations_config
1203
-
1204
- :returns: Locations configuration dict
1205
- :rtype: dict
1206
- """
1207
- user_config_locations_list = self.eodag_api.locations_config
1208
-
1209
- locations_config_model = deepcopy(self.stac_config["locations_catalogs"])
1210
-
1211
- locations_config: Dict[str, str] = {}
1212
- for loc in user_config_locations_list:
1213
- # parse jsonpath
1214
- parsed = jsonpath_parse_dict_items(
1215
- locations_config_model, {"shp_location": loc}
1216
- )
1217
-
1218
- # set default child/parent for this location
1219
- parsed["location"]["parent_key"] = "{}_list".format(loc["name"])
1220
-
1221
- locations_config["{}_list".format(loc["name"])] = parsed["locations_list"]
1222
- locations_config[loc["name"]] = parsed["location"]
1223
-
1224
- return locations_config
1225
-
1226
- def __build_stac_catalog(
1227
- self, catalogs: List[str] = [], fetch_providers: bool = True
1228
- ) -> StacCatalog:
1229
- """Build nested catalog from catalag list
1230
-
1231
- :param catalogs: (optional) Catalogs list
1232
- :type catalogs: list
1233
- :param fetch_providers: (optional) Whether to fetch providers for new product
1234
- types or not
1235
- :type fetch_providers: bool
1236
- :returns: This catalog obj
1237
- :rtype: :class:`eodag.stac.StacCatalog`
1238
- """
1239
- # update conf with user shp locations
1240
- locations_config = self.build_locations_config()
1241
-
1242
- self.stac_config["catalogs"] = dict(
1243
- deepcopy(self.stac_config["catalogs"]), **locations_config
1244
- )
1245
-
1246
- if len(catalogs) == 0:
1247
- # Build root catalog combined with landing page
1248
- self.__update_data_from_catalog_config(
1249
- {
1250
- "model": dict(
1251
- deepcopy(self.stac_config["landing_page"]),
1252
- **{"provider": self.provider},
1253
- )
1254
- }
1255
- )
1256
-
1257
- # build children : product_types
1258
- product_types_list = [
1259
- pt
1260
- for pt in self.eodag_api.list_product_types(
1261
- provider=self.provider, fetch_providers=fetch_providers
1262
- )
1263
- ]
1264
- self.set_children(
1265
- [
1266
- {
1267
- "rel": "child",
1268
- "href": urljoin(
1269
- self.url, f"{STAC_CATALOGS_PREFIX}/{product_type['ID']}"
1270
- ),
1271
- "title": product_type["ID"],
1272
- }
1273
- for product_type in product_types_list
1274
- ]
1275
- )
1276
- else:
1277
- # use product_types_list as base for building nested catalogs
1278
- self.__update_data_from_catalog_config(
1279
- deepcopy(self.stac_config["catalogs"]["product_types_list"])
1280
- )
1281
-
1282
- for idx, cat in enumerate(catalogs):
1283
- if idx % 2 == 0:
1284
- # even: cat is a filtering value ----------------------------------
1285
- cat_data_name = self.catalog_config["child_key"]
1286
- cat_data_value = cat
1287
-
1288
- # update data
1289
- set_data_method_name = (
1290
- "set_stac_%s_by_id" % cat_data_name
1291
- if "catalog_type"
1292
- not in self.stac_config["catalogs"][cat_data_name].keys()
1293
- else "set_stac_%s_by_id"
1294
- % self.stac_config["catalogs"][cat_data_name]["catalog_type"]
1295
- )
1296
- set_data_method = getattr(self, set_data_method_name)
1297
- set_data_method(cat_data_value, catalog_name=cat_data_name)
1298
-
1299
- if idx == len(catalogs) - 1:
1300
- # build children : remaining filtering keys
1301
- remaining_catalogs_list = [
1302
- c
1303
- for c in self.stac_config["catalogs"].keys()
1304
- # keep filters not used yet AND
1305
- if self.stac_config["catalogs"][c]["model"]["id"]
1306
- not in catalogs
1307
- and (
1308
- # filters with no parent_key constraint (no key, or key=None) OR
1309
- "parent_key" not in self.stac_config["catalogs"][c]
1310
- or not self.stac_config["catalogs"][c]["parent_key"]
1311
- # filters matching parent_key constraint
1312
- or self.stac_config["catalogs"][c]["parent_key"]
1313
- == cat_data_name
1314
- )
1315
- # AND filters that match parent attr constraint (locations)
1316
- and (
1317
- "parent" not in self.stac_config["catalogs"][c]
1318
- or not self.stac_config["catalogs"][c]["parent"]["key"]
1319
- or (
1320
- self.stac_config["catalogs"][c]["parent"]["key"]
1321
- == cat_data_name
1322
- and self.stac_config["catalogs"][c]["parent"]["attr"]
1323
- == cat_data_value
1324
- )
1325
- )
1326
- ]
1327
-
1328
- self.set_children(
1329
- [
1330
- {
1331
- "rel": "child",
1332
- "href": self.url
1333
- + "/"
1334
- + self.stac_config["catalogs"][c]["model"]["id"],
1335
- "title": str(
1336
- self.stac_config["catalogs"][c]["model"]["id"]
1337
- ),
1338
- }
1339
- for c in remaining_catalogs_list
1340
- ]
1341
- + [
1342
- {
1343
- "rel": "items",
1344
- "href": self.url + "/items",
1345
- "title": "items",
1346
- }
1347
- ]
1348
- )
1349
-
1350
- else:
1351
- # odd: cat is a filtering key -------------------------------------
1352
- try:
1353
- cat_key = [
1354
- c
1355
- for c in self.stac_config["catalogs"].keys()
1356
- if self.stac_config["catalogs"][c]["model"]["id"] == cat
1357
- ][0]
1358
- except IndexError:
1359
- raise ValidationError(
1360
- "Bad settings for %s in stac_config catalogs" % cat
1361
- )
1362
- cat_config = deepcopy(self.stac_config["catalogs"][cat_key])
1363
- # update data
1364
- self.__update_data_from_catalog_config(cat_config)
1365
-
1366
- # get filtering values list
1367
- get_data_method_name = (
1368
- "get_stac_%s" % cat_key
1369
- if "catalog_type"
1370
- not in self.stac_config["catalogs"][cat_key].keys()
1371
- else "get_stac_%s"
1372
- % self.stac_config["catalogs"][cat_key]["catalog_type"]
1373
- )
1374
- get_data_method = getattr(self, get_data_method_name)
1375
- cat_data_list = get_data_method(catalog_name=cat_key)
1376
-
1377
- if idx == len(catalogs) - 1:
1378
- # filtering values list as children (do not include items)
1379
- self.set_children(
1380
- [
1381
- {
1382
- "rel": "child",
1383
- "href": self.url + "/" + str(filtering_data),
1384
- "title": str(filtering_data),
1385
- }
1386
- for filtering_data in cat_data_list
1387
- ]
1388
- )
1389
-
1390
- return self
1391
-
1392
- def get_stac_catalog(self) -> Dict[str, Any]:
1393
- """Get nested STAC catalog as data dict
1394
-
1395
- :returns: Catalog dictionnary
1396
- :rtype: dict
1397
- """
1398
- return self.as_dict()