eodag 3.9.1__py3-none-any.whl → 4.0.0a1__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (71) hide show
  1. eodag/api/core.py +378 -419
  2. eodag/api/product/__init__.py +3 -3
  3. eodag/api/product/_product.py +68 -40
  4. eodag/api/product/drivers/__init__.py +3 -5
  5. eodag/api/product/drivers/base.py +1 -18
  6. eodag/api/product/metadata_mapping.py +151 -215
  7. eodag/api/search_result.py +13 -7
  8. eodag/cli.py +72 -395
  9. eodag/config.py +46 -50
  10. eodag/plugins/apis/base.py +2 -2
  11. eodag/plugins/apis/ecmwf.py +20 -21
  12. eodag/plugins/apis/usgs.py +37 -33
  13. eodag/plugins/authentication/aws_auth.py +36 -1
  14. eodag/plugins/authentication/base.py +18 -3
  15. eodag/plugins/authentication/sas_auth.py +15 -0
  16. eodag/plugins/crunch/filter_date.py +3 -3
  17. eodag/plugins/crunch/filter_latest_intersect.py +2 -2
  18. eodag/plugins/crunch/filter_latest_tpl_name.py +1 -1
  19. eodag/plugins/download/aws.py +45 -41
  20. eodag/plugins/download/base.py +13 -14
  21. eodag/plugins/download/http.py +65 -65
  22. eodag/plugins/manager.py +28 -29
  23. eodag/plugins/search/__init__.py +3 -4
  24. eodag/plugins/search/base.py +128 -77
  25. eodag/plugins/search/build_search_result.py +105 -107
  26. eodag/plugins/search/cop_marine.py +44 -47
  27. eodag/plugins/search/csw.py +33 -33
  28. eodag/plugins/search/qssearch.py +335 -354
  29. eodag/plugins/search/stac_list_assets.py +1 -1
  30. eodag/plugins/search/static_stac_search.py +31 -31
  31. eodag/resources/{product_types.yml → collections.yml} +2353 -2429
  32. eodag/resources/ext_collections.json +1 -0
  33. eodag/resources/ext_product_types.json +1 -1
  34. eodag/resources/providers.yml +2432 -2714
  35. eodag/resources/stac_provider.yml +46 -90
  36. eodag/types/queryables.py +55 -91
  37. eodag/types/search_args.py +1 -1
  38. eodag/utils/__init__.py +94 -21
  39. eodag/utils/exceptions.py +6 -6
  40. eodag/utils/free_text_search.py +3 -3
  41. {eodag-3.9.1.dist-info → eodag-4.0.0a1.dist-info}/METADATA +11 -88
  42. eodag-4.0.0a1.dist-info/RECORD +92 -0
  43. {eodag-3.9.1.dist-info → eodag-4.0.0a1.dist-info}/entry_points.txt +0 -4
  44. eodag/plugins/authentication/oauth.py +0 -60
  45. eodag/plugins/download/creodias_s3.py +0 -64
  46. eodag/plugins/download/s3rest.py +0 -351
  47. eodag/plugins/search/data_request_search.py +0 -565
  48. eodag/resources/stac.yml +0 -294
  49. eodag/resources/stac_api.yml +0 -2105
  50. eodag/rest/__init__.py +0 -24
  51. eodag/rest/cache.py +0 -70
  52. eodag/rest/config.py +0 -67
  53. eodag/rest/constants.py +0 -26
  54. eodag/rest/core.py +0 -764
  55. eodag/rest/errors.py +0 -210
  56. eodag/rest/server.py +0 -604
  57. eodag/rest/server.wsgi +0 -6
  58. eodag/rest/stac.py +0 -1032
  59. eodag/rest/templates/README +0 -1
  60. eodag/rest/types/__init__.py +0 -18
  61. eodag/rest/types/collections_search.py +0 -44
  62. eodag/rest/types/eodag_search.py +0 -386
  63. eodag/rest/types/queryables.py +0 -174
  64. eodag/rest/types/stac_search.py +0 -272
  65. eodag/rest/utils/__init__.py +0 -207
  66. eodag/rest/utils/cql_evaluate.py +0 -119
  67. eodag/rest/utils/rfc3339.py +0 -64
  68. eodag-3.9.1.dist-info/RECORD +0 -115
  69. {eodag-3.9.1.dist-info → eodag-4.0.0a1.dist-info}/WHEEL +0 -0
  70. {eodag-3.9.1.dist-info → eodag-4.0.0a1.dist-info}/licenses/LICENSE +0 -0
  71. {eodag-3.9.1.dist-info → eodag-4.0.0a1.dist-info}/top_level.txt +0 -0
eodag/rest/stac.py DELETED
@@ -1,1032 +0,0 @@
1
- # -*- coding: utf-8 -*-
2
- # Copyright 2018, CS Systemes d'Information, https://www.csgroup.eu/
3
- #
4
- # This file is part of EODAG project
5
- # https://www.github.com/CS-SI/EODAG
6
- #
7
- # Licensed under the Apache License, Version 2.0 (the "License");
8
- # you may not use this file except in compliance with the License.
9
- # You may obtain a copy of the License at
10
- #
11
- # http://www.apache.org/licenses/LICENSE-2.0
12
- #
13
- # Unless required by applicable law or agreed to in writing, software
14
- # distributed under the License is distributed on an "AS IS" BASIS,
15
- # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16
- # See the License for the specific language governing permissions and
17
- # limitations under the License.
18
- from __future__ import annotations
19
-
20
- import logging
21
- import os
22
- import warnings
23
- from collections import defaultdict
24
- from datetime import datetime, timezone
25
- from typing import TYPE_CHECKING, Any, Optional
26
- from urllib.parse import (
27
- parse_qs,
28
- quote,
29
- urlencode,
30
- urlparse,
31
- urlsplit,
32
- urlunparse,
33
- urlunsplit,
34
- )
35
-
36
- import geojson
37
- from jsonpath_ng.jsonpath import Child
38
- from shapely.geometry import Polygon
39
-
40
- from eodag.api.product.metadata_mapping import (
41
- DEFAULT_METADATA_MAPPING,
42
- format_metadata,
43
- get_metadata_path,
44
- )
45
- from eodag.rest.config import Settings
46
- from eodag.rest.types.stac_search import SearchPostRequest
47
- from eodag.rest.utils.rfc3339 import str_to_interval
48
- from eodag.utils import (
49
- deepcopy,
50
- dict_items_recursive_apply,
51
- format_dict_items,
52
- guess_file_type,
53
- jsonpath_parse_dict_items,
54
- string_to_jsonpath,
55
- update_nested_dict,
56
- )
57
- from eodag.utils.exceptions import (
58
- NoMatchingProductType,
59
- NotAvailableError,
60
- RequestError,
61
- TimeOutError,
62
- ValidationError,
63
- )
64
- from eodag.utils.requests import fetch_json
65
-
66
- if TYPE_CHECKING:
67
- from eodag.api.core import EODataAccessGateway
68
- from eodag.api.product import EOProduct
69
- from eodag.api.search_result import SearchResult
70
-
71
-
72
- warnings.warn(
73
- "The module `eodag.rest.stac` is deprecated since v3.9.0 and will be removed in a future version. "
74
- "The STAC server has moved to https://github.com/CS-SI/stac-fastapi-eodag",
75
- category=DeprecationWarning,
76
- stacklevel=2,
77
- )
78
-
79
- logger = logging.getLogger("eodag.rest.stac")
80
-
81
- # fields not to put in item properties
82
- COLLECTION_PROPERTIES = [
83
- "abstract",
84
- "instrument",
85
- "platform",
86
- "platformSerialIdentifier",
87
- "processingLevel",
88
- "sensorType",
89
- "md5",
90
- "license",
91
- "title",
92
- "missionStartDate",
93
- "missionEndDate",
94
- "keywords",
95
- "stacCollection",
96
- "alias",
97
- "productType",
98
- ]
99
- IGNORED_ITEM_PROPERTIES = [
100
- "_id",
101
- "id",
102
- "keyword",
103
- "quicklook",
104
- "thumbnail",
105
- "downloadLink",
106
- "orderLink",
107
- "_dc_qs",
108
- "qs",
109
- "defaultGeometry",
110
- "_date",
111
- "productType",
112
- ]
113
-
114
-
115
- def _quote_url_path(url: str) -> str:
116
- parsed = urlsplit(url)
117
- path = quote(parsed.path)
118
- components = (parsed.scheme, parsed.netloc, path, parsed.query, parsed.fragment)
119
- return urlunsplit(components)
120
-
121
-
122
- class StacCommon:
123
- """Stac common object
124
-
125
- :param url: Requested URL
126
- :param stac_config: STAC configuration from stac.yml conf file
127
- :param provider: (optional) Chosen provider
128
- :param eodag_api: EODAG python API instance
129
- :param root: (optional) API root
130
- """
131
-
132
- def __init__(
133
- self,
134
- url: str,
135
- stac_config: dict[str, Any],
136
- provider: Optional[str],
137
- eodag_api: EODataAccessGateway,
138
- root: str = "/",
139
- ) -> None:
140
- self.url = url.rstrip("/") if len(url) > 1 else url
141
- self.stac_config = stac_config
142
- self.provider = provider
143
- self.eodag_api = eodag_api
144
- self.root = root.rstrip("/") if len(root) > 1 else root
145
-
146
- self.data: dict[str, Any] = {}
147
-
148
- def update_data(self, data: dict[str, Any]) -> None:
149
- """Updates data using given input STAC dict data
150
-
151
- :param data: Catalog data (parsed STAC dict)
152
- """
153
- self.data.update(data)
154
-
155
- # bbox: str to float
156
- if (
157
- "extent" in self.data.keys()
158
- and "spatial" in self.data["extent"].keys()
159
- and "bbox" in self.data["extent"]["spatial"].keys()
160
- ):
161
- for i, bbox in enumerate(self.data["extent"]["spatial"]["bbox"]):
162
- self.data["extent"]["spatial"]["bbox"][i] = [float(x) for x in bbox]
163
-
164
- def apply_method_none(_: str, v: str) -> Optional[str]:
165
- """ "None" values to None"""
166
- return None if v == "None" else v
167
-
168
- self.data = dict_items_recursive_apply(self.data, apply_method_none)
169
-
170
- def apply_method_ids(k, v):
171
- """ids and titles as str"""
172
- return str(v) if k in ["title", "id"] else v
173
-
174
- self.data = dict_items_recursive_apply(self.data, apply_method_ids)
175
-
176
- # empty stac_extensions: "" to []
177
- if not self.data.get("stac_extensions", True):
178
- self.data["stac_extensions"] = []
179
-
180
- @staticmethod
181
- def get_stac_extension(
182
- url: str, stac_config: dict[str, Any], extension: str, **kwargs: Any
183
- ) -> dict[str, str]:
184
- """Parse STAC extension from config and return as dict
185
-
186
- :param url: Requested URL
187
- :param stac_config: STAC configuration from stac.yml conf file
188
- :param extension: Extension name
189
- :param kwargs: Additional variables needed for parsing extension
190
- :returns: STAC extension as dictionary
191
- """
192
- extension_model = deepcopy(stac_config).get("extensions", {}).get(extension, {})
193
-
194
- # parse f-strings
195
- format_args = deepcopy(stac_config)
196
- format_args["extension"] = {
197
- "url": url,
198
- "properties": kwargs.get("properties", {}),
199
- }
200
- return format_dict_items(extension_model, **format_args)
201
-
202
- def get_provider_dict(self, provider: str) -> dict[str, Any]:
203
- """Generate STAC provider dict"""
204
- provider_config = next(
205
- p
206
- for p in self.eodag_api.providers_config.values()
207
- if provider in [p.name, getattr(p, "group", None)]
208
- )
209
- return {
210
- "name": getattr(provider_config, "group", provider_config.name),
211
- "description": getattr(provider_config, "description", None),
212
- "roles": getattr(provider_config, "roles", None),
213
- "url": getattr(provider_config, "url", None),
214
- "priority": getattr(provider_config, "priority", None),
215
- }
216
-
217
-
218
- class StacItem(StacCommon):
219
- """Stac item object
220
-
221
- :param url: Requested URL
222
- :param stac_config: STAC configuration from stac.yml conf file
223
- :param provider: (optional) Chosen provider
224
- :param eodag_api: EODAG python API instance
225
- :param root: (optional) API root
226
- """
227
-
228
- def __init__(
229
- self,
230
- url: str,
231
- stac_config: dict[str, Any],
232
- provider: Optional[str],
233
- eodag_api: EODataAccessGateway,
234
- root: str = "/",
235
- ) -> None:
236
- super(StacItem, self).__init__(
237
- url=url,
238
- stac_config=stac_config,
239
- provider=provider,
240
- eodag_api=eodag_api,
241
- root=root,
242
- )
243
-
244
- def __get_item_list(
245
- self, search_results: SearchResult, catalog: dict[str, Any]
246
- ) -> list[dict[str, Any]]:
247
- """Build STAC items list from EODAG search results
248
-
249
- :param search_results: EODAG search results
250
- :param catalog: STAC catalog dict used for parsing item metadata
251
- :returns: STAC item dicts list
252
- """
253
- if len(search_results) <= 0:
254
- return []
255
-
256
- item_model = self.__filter_item_model_properties(
257
- self.stac_config["item"], str(search_results[0].product_type)
258
- )
259
-
260
- # check if some items need to be converted
261
- need_conversion: dict[str, Any] = {}
262
- for k, v in item_model["properties"].items():
263
- if isinstance(v, str):
264
- conversion, item_model["properties"][k] = get_metadata_path(
265
- item_model["properties"][k]
266
- )
267
- if conversion is not None:
268
- need_conversion[k] = conversion
269
- # convert str to jsonpath if needed
270
- item_model["properties"][k] = string_to_jsonpath(
271
- k, item_model["properties"][k]
272
- )
273
-
274
- item_props = [
275
- p.right.fields[0]
276
- for p in item_model["properties"].values()
277
- if isinstance(p, Child)
278
- ]
279
- ignored_props = COLLECTION_PROPERTIES + item_props + IGNORED_ITEM_PROPERTIES
280
-
281
- item_list: list[dict[str, Any]] = []
282
- for product in search_results:
283
- product_dict = deepcopy(product.__dict__)
284
-
285
- product_item: dict[str, Any] = jsonpath_parse_dict_items(
286
- item_model,
287
- {
288
- "product": product_dict,
289
- "providers": [self.get_provider_dict(product.provider)],
290
- },
291
- )
292
-
293
- # add additional item props
294
- for p in set(product.properties) - set(ignored_props):
295
- prefix = getattr(
296
- self.eodag_api.providers_config[product.provider],
297
- "group",
298
- product.provider,
299
- )
300
- key = p if ":" in p else f"{prefix}:{p}"
301
- product_item["properties"][key] = product.properties[p]
302
-
303
- # parse download link
304
- downloadlink_href = (
305
- f"{catalog['url']}/items/{product.properties['id']}/download"
306
- )
307
- _dc_qs = product.properties.get("_dc_qs")
308
- url_parts = urlparse(downloadlink_href)
309
- query_dict = parse_qs(url_parts.query)
310
- without_arg_url = (
311
- f"{url_parts.scheme}://{url_parts.netloc}{url_parts.path}"
312
- if url_parts.scheme
313
- else f"{url_parts.netloc}{url_parts.path}"
314
- )
315
- # add provider to query-args
316
- p_config = self.eodag_api.providers_config[product.provider]
317
- query_dict.update(provider=[getattr(p_config, "group", p_config.name)])
318
- # add datacube query-string to query-args
319
- if _dc_qs:
320
- query_dict.update(_dc_qs=[_dc_qs])
321
- if query_dict:
322
- downloadlink_href = (
323
- f"{without_arg_url}?{urlencode(query_dict, doseq=True)}"
324
- )
325
-
326
- # generate STAC assets
327
- product_item["assets"] = self._get_assets(
328
- product, downloadlink_href, without_arg_url, query_dict, _dc_qs
329
- )
330
-
331
- # apply conversion if needed
332
- for prop_key, prop_val in need_conversion.items():
333
- conv_func, conv_args = prop_val
334
- # colon `:` in key breaks format() method, hide it
335
- formatable_prop_key = prop_key.replace(":", "")
336
- if conv_args is not None:
337
- product_item["properties"][prop_key] = format_metadata(
338
- "{%s#%s(%s)}" % (formatable_prop_key, conv_func, conv_args),
339
- **{formatable_prop_key: product_item["properties"][prop_key]},
340
- )
341
- else:
342
- product_item["properties"][prop_key] = format_metadata(
343
- "{%s#%s}" % (formatable_prop_key, conv_func),
344
- **{formatable_prop_key: product_item["properties"][prop_key]},
345
- )
346
-
347
- # parse f-strings
348
- format_args = deepcopy(self.stac_config)
349
- format_args["catalog"] = catalog
350
- format_args["item"] = product_item
351
- product_item = format_dict_items(product_item, **format_args)
352
- product_item["bbox"] = [float(i) for i in product_item["bbox"]]
353
-
354
- # transform shapely geometry to geojson
355
- product_item["geometry"] = geojson.loads(
356
- geojson.dumps(product_item["geometry"])
357
- )
358
-
359
- # remove empty properties
360
- product_item = self.__filter_item_properties_values(product_item)
361
-
362
- # quote invalid characters in links
363
- for link in product_item["links"]:
364
- link["href"] = _quote_url_path(link["href"])
365
-
366
- # update item link with datacube query-string
367
- if _dc_qs or self.provider:
368
- url_parts = urlparse(str(product_item["links"][0]["href"]))
369
- without_arg_url = (
370
- f"{url_parts.scheme}://{url_parts.netloc}{url_parts.path}"
371
- if url_parts.scheme
372
- else f"{url_parts.netloc}{url_parts.path}"
373
- )
374
- product_item["links"][0][
375
- "href"
376
- ] = f"{without_arg_url}?{urlencode(query_dict, doseq=True)}"
377
-
378
- item_list.append(product_item)
379
-
380
- return item_list
381
-
382
- def _get_assets(
383
- self,
384
- product: EOProduct,
385
- downloadlink_href: str,
386
- without_arg_url: str,
387
- query_dict: Optional[dict[str, Any]] = None,
388
- _dc_qs: Optional[str] = None,
389
- ) -> dict[str, Any]:
390
- assets: dict[str, Any] = {}
391
- settings = Settings.from_environment()
392
-
393
- if _dc_qs:
394
- parsed = urlparse(product.remote_location)
395
- fragments = parsed.fragment.split("?")
396
- parsed = parsed._replace(fragment=f"{fragments[0]}?_dc_qs={_dc_qs}")
397
- origin_href = urlunparse(parsed)
398
- else:
399
- origin_href = product.remote_location
400
-
401
- # update download link with up-to-date query-args
402
- quoted_href = _quote_url_path(
403
- downloadlink_href
404
- ) # quote invalid characters in url
405
- assets["downloadLink"] = {
406
- "title": "Download link",
407
- "href": quoted_href,
408
- "type": "application/zip",
409
- }
410
-
411
- if not origin_href.startswith(tuple(settings.origin_url_blacklist)):
412
- assets["downloadLink"]["alternate"] = {
413
- "origin": {
414
- "title": "Origin asset link",
415
- "href": origin_href,
416
- }
417
- }
418
-
419
- if "storageStatus" in product.properties:
420
- assets["downloadLink"]["storage:tier"] = product.properties["storageStatus"]
421
-
422
- # move origin asset urls to alternate links and replace with eodag-server ones
423
- if product.assets:
424
- origin_assets = product.assets.as_dict()
425
- # replace origin asset urls with eodag-server ones
426
- for asset_key, asset_value in origin_assets.items():
427
- # use origin asset as default
428
- assets[asset_key] = asset_value
429
- # origin assets as alternate link
430
- if not asset_value["href"].startswith(
431
- tuple(settings.origin_url_blacklist)
432
- ):
433
- assets[asset_key]["alternate"] = {
434
- "origin": {
435
- "title": "Origin asset link",
436
- "href": asset_value["href"],
437
- }
438
- }
439
- # use server-mode assets download links
440
- asset_value["href"] = without_arg_url
441
- if query_dict:
442
- assets[asset_key][
443
- "href"
444
- ] += f"/{asset_key}?{urlencode(query_dict, doseq=True)}"
445
- else:
446
- assets[asset_key]["href"] += f"/{asset_key}"
447
- if asset_type := asset_value.get("type"):
448
- assets[asset_key]["type"] = asset_type
449
- if origin := assets[asset_key].get("alternate", {}).get("origin"):
450
- origin["type"] = asset_type
451
- asset_value["href"] = _quote_url_path(asset_value["href"])
452
-
453
- if thumbnail_url := product.properties.get(
454
- "quicklook", product.properties.get("thumbnail")
455
- ):
456
- assets["thumbnail"] = {
457
- "title": "Thumbnail",
458
- "href": thumbnail_url,
459
- "role": "thumbnail",
460
- }
461
- if mime_type := guess_file_type(thumbnail_url):
462
- assets["thumbnail"]["type"] = mime_type
463
- return assets
464
-
465
- def get_stac_items(
466
- self,
467
- search_results: SearchResult,
468
- total: int,
469
- catalog: dict[str, Any],
470
- next_link: Optional[dict[str, Any]],
471
- ) -> dict[str, Any]:
472
- """Build STAC items from EODAG search results
473
-
474
- :param search_results: EODAG search results
475
- :param catalog: STAC catalog dict used for parsing item metadata
476
- :returns: Items dictionary
477
- """
478
- items_model = deepcopy(self.stac_config["items"])
479
-
480
- if "?" in self.url:
481
- # search endpoint: use page url as self link
482
- for i, _ in enumerate(items_model["links"]):
483
- if items_model["links"][i]["rel"] == "self":
484
- items_model["links"][i]["href"] = catalog["url"]
485
-
486
- timestamp = datetime.now(timezone.utc).isoformat().replace("+00:00", "Z")
487
-
488
- # parse jsonpath
489
- items = jsonpath_parse_dict_items(
490
- items_model,
491
- {
492
- "numberMatched": total,
493
- "numberReturned": len(search_results),
494
- "timeStamp": timestamp,
495
- },
496
- )
497
- # parse f-strings
498
- format_args = deepcopy(self.stac_config)
499
- format_args["catalog"] = catalog
500
- items = format_dict_items(items, **format_args)
501
-
502
- if next_link:
503
- items["links"].append(next_link)
504
-
505
- # provide static catalog to build features
506
- if "search?" in catalog["url"]:
507
- catalog["url"] = os.path.join(
508
- catalog["url"].split("search?")[0],
509
- "collections",
510
- catalog["id"],
511
- )
512
- else:
513
- catalog["url"] = catalog["url"].split("?")[0]
514
- items["features"] = self.__get_item_list(search_results, catalog)
515
-
516
- self.update_data(items)
517
- return self.data
518
-
519
- def __filter_item_model_properties(
520
- self, item_model: dict[str, Any], product_type: str
521
- ) -> dict[str, Any]:
522
- """Filter item model depending on product type metadata and its extensions.
523
- Removes not needed parameters, and adds supplementary ones as
524
- part of oseo extension.
525
-
526
- :param item_model: Item model from stac_config
527
- :param product_type: Product type
528
- :returns: Filtered item model
529
- """
530
- try:
531
- product_type_dict = [
532
- pt
533
- for pt in self.eodag_api.list_product_types(
534
- provider=self.provider, fetch_providers=False
535
- )
536
- if pt["ID"] == product_type
537
- or ("alias" in pt and pt["alias"] == product_type)
538
- ][0]
539
- except IndexError as e:
540
- raise NoMatchingProductType(
541
- f"Product type {product_type} not available for {self.provider}"
542
- ) from e
543
-
544
- result_item_model = deepcopy(item_model)
545
- result_item_model["stac_extensions"] = list(
546
- self.stac_config["stac_extensions"].values()
547
- )
548
-
549
- # build jsonpath for eodag product default properties and adapt path
550
- eodag_properties_dict = {
551
- k: string_to_jsonpath(k, v.replace("$.", "$.product."))
552
- for k, v in DEFAULT_METADATA_MAPPING.items()
553
- if "$.properties." in v
554
- }
555
- # add missing properties as oseo:missingProperty
556
- for k, v in eodag_properties_dict.items():
557
- if (
558
- v not in result_item_model["properties"].values()
559
- and k not in self.stac_config["metadata_ignore"]
560
- and not any(
561
- k in str(prop) for prop in result_item_model["properties"].values()
562
- )
563
- ):
564
- result_item_model["properties"]["oseo:" + k] = string_to_jsonpath(k, v)
565
-
566
- # Filter out unneeded extensions
567
- if product_type_dict.get("sensorType", "RADAR") != "RADAR":
568
- result_item_model["stac_extensions"].remove(
569
- self.stac_config["stac_extensions"]["sar"]
570
- )
571
-
572
- # Filter out unneeded properties
573
- extensions_prefixes = [
574
- k
575
- for k, v in self.stac_config["stac_extensions"].items()
576
- if v in result_item_model["stac_extensions"]
577
- ]
578
- for k, v in item_model["properties"].items():
579
- # remove key if extension not in stac_extensions
580
- if ":" in k and k.split(":")[0] not in extensions_prefixes:
581
- result_item_model["properties"].pop(k, None)
582
-
583
- return result_item_model
584
-
585
- def __filter_item_properties_values(self, item: dict[str, Any]) -> dict[str, Any]:
586
- """Removes empty properties, unused extensions, and add missing extensions
587
-
588
- :param item: STAC item data
589
- :returns: Filtered item model
590
- """
591
- all_extensions_dict: dict[str, str] = deepcopy(
592
- self.stac_config["stac_extensions"]
593
- )
594
- # parse f-strings with root
595
- all_extensions_dict = format_dict_items(
596
- all_extensions_dict, **{"catalog": {"root": self.root}}
597
- )
598
-
599
- item["stac_extensions"] = []
600
- # dict to list of keys to permit pop() while iterating
601
- for k in list(item["properties"]):
602
- extension_prefix: str = k.split(":")[0]
603
-
604
- if item["properties"][k] is None:
605
- item["properties"].pop(k, None)
606
- # feed found extensions list
607
- elif (
608
- extension_prefix in all_extensions_dict.keys()
609
- and all_extensions_dict[extension_prefix] not in item["stac_extensions"]
610
- ):
611
- # append path from item extensions
612
- item["stac_extensions"].append(all_extensions_dict[extension_prefix])
613
-
614
- return item
615
-
616
- def get_stac_item_from_product(self, product: EOProduct) -> dict[str, Any]:
617
- """Build STAC item from EODAG product
618
-
619
- :param product: EODAG product
620
- :returns: STAC item
621
- """
622
- product_type = str(product.product_type)
623
-
624
- item_model = self.__filter_item_model_properties(
625
- self.stac_config["item"], product_type
626
- )
627
-
628
- catalog = StacCatalog(
629
- url=self.url.split("/items")[0],
630
- stac_config=self.stac_config,
631
- root=self.root,
632
- provider=self.provider,
633
- eodag_api=self.eodag_api,
634
- collection=product_type,
635
- )
636
-
637
- product_dict = deepcopy(product.__dict__)
638
- product_dict["assets"] = product.assets.as_dict()
639
-
640
- # parse jsonpath
641
- product_item = jsonpath_parse_dict_items(
642
- item_model,
643
- {
644
- "product": product_dict,
645
- "providers": [self.get_provider_dict(product.provider)],
646
- },
647
- )
648
- # parse f-strings
649
- format_args = deepcopy(self.stac_config)
650
- format_args["catalog"] = {
651
- **catalog.data,
652
- **{"url": catalog.url, "root": catalog.root},
653
- }
654
- format_args["item"] = product_item
655
- product_item = format_dict_items(product_item, **format_args)
656
- product_item["bbox"] = [float(i) for i in product_item["bbox"]]
657
-
658
- # remove empty properties
659
- product_item = self.__filter_item_properties_values(product_item)
660
-
661
- self.update_data(product_item)
662
- return self.data
663
-
664
-
665
- class StacCollection(StacCommon):
666
- """Stac collection object
667
-
668
- :param url: Requested URL
669
- :param stac_config: STAC configuration from stac.yml conf file
670
- :param provider: (optional) Chosen provider
671
- :param eodag_api: EODAG python API instance
672
- :param root: (optional) API root
673
- """
674
-
675
- # External STAC collections
676
- ext_stac_collections: dict[str, dict[str, Any]] = dict()
677
-
678
- @classmethod
679
- def fetch_external_stac_collections(cls, eodag_api: EODataAccessGateway) -> None:
680
- """Load external STAC collections
681
-
682
- :param eodag_api: EODAG python API instance
683
- """
684
- list_product_types = eodag_api.list_product_types(fetch_providers=False)
685
- for product_type in list_product_types:
686
- ext_stac_collection_path = product_type.get("stacCollection")
687
- if not ext_stac_collection_path:
688
- continue
689
- logger.info(f"Fetching external STAC collection for {product_type['ID']}")
690
-
691
- try:
692
- ext_stac_collection = fetch_json(ext_stac_collection_path)
693
- except (RequestError, TimeOutError) as e:
694
- logger.debug(e)
695
- logger.warning(
696
- f"Could not read remote external STAC collection from {ext_stac_collection_path}",
697
- )
698
- ext_stac_collection = {}
699
-
700
- cls.ext_stac_collections[product_type["ID"]] = ext_stac_collection
701
-
702
- def __init__(
703
- self,
704
- url: str,
705
- stac_config: dict[str, Any],
706
- provider: Optional[str],
707
- eodag_api: EODataAccessGateway,
708
- root: str = "/",
709
- ) -> None:
710
- super(StacCollection, self).__init__(
711
- url=url,
712
- stac_config=stac_config,
713
- provider=provider,
714
- eodag_api=eodag_api,
715
- root=root,
716
- )
717
-
718
- def __list_product_type_providers(self, product_type: dict[str, Any]) -> list[str]:
719
- """Retrieve a list of providers for a given product type.
720
-
721
- :param product_type: Dictionary containing information about the product type.
722
- :return: A list of provider names.
723
- """
724
- if self.provider:
725
- return [self.provider]
726
-
727
- return [
728
- plugin.provider
729
- for plugin in self.eodag_api._plugins_manager.get_search_plugins(
730
- product_type=product_type.get("_id", product_type["ID"])
731
- )
732
- ]
733
-
734
- def __generate_stac_collection(
735
- self, collection_model: Any, product_type: dict[str, Any]
736
- ) -> dict[str, Any]:
737
- """Generate a STAC collection dictionary for a given product type.
738
-
739
- :param collection_model: The base model for the STAC collection.
740
- :param product_type: Dictionary containing information about the product type.
741
- :return: A dictionary representing the STAC collection for the given product type.
742
- """
743
- providers = self.__list_product_type_providers(product_type)
744
-
745
- providers_dict: dict[str, dict[str, Any]] = {}
746
- for provider in providers:
747
- p_dict = self.get_provider_dict(provider)
748
- providers_dict.setdefault(p_dict["name"], p_dict)
749
- providers_list = list(providers_dict.values())
750
-
751
- # parse jsonpath
752
- product_type_collection = jsonpath_parse_dict_items(
753
- collection_model,
754
- {
755
- "product_type": product_type,
756
- "providers": providers_list,
757
- },
758
- )
759
- # override EODAG's collection with the external collection
760
- ext_stac_collection = deepcopy(
761
- self.ext_stac_collections.get(product_type["ID"], {})
762
- )
763
-
764
- # update links (keep eodag links as defaults)
765
- ext_stac_collection.setdefault("links", {})
766
- for link in product_type_collection["links"]:
767
- ext_stac_collection["links"] = [
768
- x for x in ext_stac_collection["links"] if x["rel"] != link["rel"]
769
- ]
770
- ext_stac_collection["links"].append(link)
771
-
772
- # merge "summaries"
773
- ext_stac_collection["summaries"] = {
774
- k: v
775
- for k, v in {
776
- **ext_stac_collection.get("summaries", {}),
777
- **product_type_collection["summaries"],
778
- }.items()
779
- if v and any(v)
780
- }
781
-
782
- # merge "keywords" lists
783
- try:
784
- ext_stac_collection["keywords"] = [
785
- k
786
- for k in set(
787
- ext_stac_collection.get("keywords", [])
788
- + product_type_collection["keywords"]
789
- )
790
- if k is not None
791
- ]
792
- except TypeError as e:
793
- logger.warning(
794
- f"Could not merge keywords from external collection for {product_type['ID']}: {str(e)}"
795
- )
796
- logger.debug(
797
- f"External collection keywords: {str(ext_stac_collection.get('keywords'))}, ",
798
- f"Product type keywords: {str(product_type_collection['keywords'])}",
799
- )
800
-
801
- product_type_collection.update(ext_stac_collection)
802
-
803
- # parse f-strings
804
- format_args = deepcopy(self.stac_config)
805
- format_args["collection"] = {
806
- **product_type_collection,
807
- **{
808
- "url": self.url
809
- if self.url.endswith(product_type["ID"])
810
- else f"{self.url}/{product_type['ID']}",
811
- "root": self.root,
812
- },
813
- }
814
- product_type_collection = format_dict_items(
815
- product_type_collection, **format_args
816
- )
817
-
818
- return product_type_collection
819
-
820
- def get_collection_list(
821
- self,
822
- collection: Optional[str] = None,
823
- q: Optional[str] = None,
824
- platform: Optional[str] = None,
825
- instrument: Optional[str] = None,
826
- constellation: Optional[str] = None,
827
- datetime: Optional[str] = None,
828
- bbox: Optional[str] = None,
829
- ) -> list[dict[str, Any]]:
830
- """Build STAC collections list
831
-
832
- :param filters: (optional) Additional filters for collections search
833
- :returns: STAC collection dicts list
834
- """
835
- collection_model = deepcopy(self.stac_config["collection"])
836
-
837
- start, end = str_to_interval(datetime)
838
-
839
- all_pt = self.eodag_api.list_product_types(
840
- provider=self.provider, fetch_providers=False
841
- )
842
-
843
- if any((collection, q, platform, instrument, constellation, datetime)):
844
- try:
845
- guessed_product_types = self.eodag_api.guess_product_type(
846
- free_text=q,
847
- platformSerialIdentifier=platform,
848
- instrument=instrument,
849
- platform=constellation,
850
- productType=collection,
851
- missionStartDate=start.isoformat() if start else None,
852
- missionEndDate=end.isoformat() if end else None,
853
- )
854
- except NoMatchingProductType:
855
- product_types = []
856
- else:
857
- product_types = [
858
- pt for pt in all_pt if pt["ID"] in guessed_product_types
859
- ]
860
- else:
861
- product_types = all_pt
862
-
863
- _bbox_poly = None
864
- if bbox:
865
- try:
866
- _bbox = [float(x) for x in bbox.split(",")]
867
- SearchPostRequest.validate_bbox(_bbox) # type: ignore
868
- _bbox_poly = Polygon.from_bounds(*_bbox)
869
- except ValueError as e:
870
- raise ValidationError(f"Wrong bbox: {e}")
871
-
872
- # list product types with all metadata using guessed ids
873
- collection_list: list[dict[str, Any]] = []
874
- for product_type in product_types:
875
- stac_collection = self.__generate_stac_collection(
876
- collection_model, product_type
877
- )
878
- # Apply bbox filter
879
- if _bbox_poly:
880
- _other_bbox_poly = Polygon.from_bounds(
881
- *stac_collection["extent"]["spatial"]["bbox"][0]
882
- )
883
- if _bbox_poly.intersects(_other_bbox_poly):
884
- collection_list.append(stac_collection)
885
- else:
886
- collection_list.append(stac_collection)
887
-
888
- return collection_list
889
-
890
-
891
- class StacCatalog(StacCommon):
892
- """Stac Catalog object
893
-
894
- :param url: Requested URL
895
- :param stac_config: STAC configuration from stac.yml conf file
896
- :param provider: Chosen provider
897
- :param eodag_api: EODAG python API instance
898
- :param root: (optional) API root
899
- :param collection: (optional) product type id
900
- """
901
-
902
- def __init__(
903
- self,
904
- url: str,
905
- stac_config: dict[str, Any],
906
- provider: Optional[str],
907
- eodag_api: EODataAccessGateway,
908
- root: str = "/",
909
- collection: Optional[str] = None,
910
- ) -> None:
911
- super(StacCatalog, self).__init__(
912
- url=url,
913
- stac_config=stac_config,
914
- provider=provider,
915
- eodag_api=eodag_api,
916
- root=root,
917
- )
918
- self.data = {}
919
-
920
- self.shp_location_config = eodag_api.locations_config
921
- self.search_args: dict[str, Any] = {}
922
- self.children: list[dict[str, Any]] = []
923
-
924
- self.catalog_config = deepcopy(stac_config["catalog"])
925
-
926
- self.__update_data_from_catalog_config({"model": {}})
927
-
928
- # expand links
929
- self.parent = "/".join(self.url.rstrip("/").split("/")[:-1])
930
- if self.parent != self.root:
931
- self.data["links"].append({"rel": "parent", "href": self.parent})
932
- if self.children:
933
- self.data["links"] += self.children
934
-
935
- # build catalog
936
- self.__build_stac_catalog(collection)
937
-
938
- def __update_data_from_catalog_config(self, catalog_config: dict[str, Any]) -> bool:
939
- """Updates configuration and data using given input catalog config
940
-
941
- :param catalog_config: Catalog config, from yml stac_config[catalogs]
942
- """
943
- model = catalog_config["model"]
944
-
945
- self.catalog_config = update_nested_dict(self.catalog_config, catalog_config)
946
-
947
- # parse f-strings
948
- # defaultdict usage will return "" for missing keys in format_args
949
- format_args = deepcopy(self.stac_config)
950
- format_args["catalog"] = defaultdict(
951
- str, dict(model, **{"root": self.root, "url": self.url})
952
- )
953
- # use existing data as parent_catalog
954
- format_args["parent_catalog"] = defaultdict(str, **self.data)
955
- parsed_model = format_dict_items(self.catalog_config["model"], **format_args)
956
-
957
- self.update_data(parsed_model)
958
-
959
- return True
960
-
961
- def __build_stac_catalog(self, collection: Optional[str] = None) -> StacCatalog:
962
- """Build nested catalog from catalag list
963
-
964
- :param collection: (optional) product type id
965
- :returns: This catalog obj
966
- """
967
- settings = Settings.from_environment()
968
-
969
- if not collection:
970
- # Build root catalog combined with landing page
971
- self.__update_data_from_catalog_config(
972
- {
973
- "model": {
974
- **deepcopy(self.stac_config["landing_page"]),
975
- **{
976
- "provider": self.provider,
977
- "id": settings.stac_api_landing_id,
978
- "title": settings.stac_api_title,
979
- "description": settings.stac_api_description,
980
- },
981
- }
982
- }
983
- )
984
- else:
985
- self.set_stac_product_type_by_id(collection)
986
- return self
987
-
988
- def set_stac_product_type_by_id(
989
- self, product_type: str, **_: Any
990
- ) -> dict[str, Any]:
991
- """Updates catalog with given product_type
992
-
993
- :param product_type: Product type
994
- """
995
- collections = StacCollection(
996
- url=self.url,
997
- stac_config=self.stac_config,
998
- provider=self.provider,
999
- eodag_api=self.eodag_api,
1000
- root=self.root,
1001
- ).get_collection_list(collection=product_type)
1002
-
1003
- if not collections:
1004
- raise NotAvailableError(f"Collection {product_type} does not exist.")
1005
-
1006
- cat_model = {
1007
- "id": "{collection[id]}",
1008
- "title": "{collection[title]}",
1009
- "description": "{collection[description]}",
1010
- "extent": "{collection[extent]}",
1011
- "crs": "http://www.opengis.net/def/crs/OGC/1.3/CRS84",
1012
- "keywords": "{collection[keywords]}",
1013
- "license": "{collection[license]}",
1014
- "providers": "{collection[providers]}",
1015
- "summaries": "{collection[summaries]}",
1016
- }
1017
- # parse f-strings
1018
- format_args = deepcopy(self.stac_config)
1019
- format_args["catalog"] = defaultdict(str, **self.data)
1020
- format_args["collection"] = collections[0]
1021
- try:
1022
- parsed_dict: dict[str, Any] = format_dict_items(cat_model, **format_args)
1023
- except Exception:
1024
- logger.error("Could not format product_type catalog")
1025
- raise
1026
-
1027
- self.update_data(parsed_dict)
1028
-
1029
- # update search args
1030
- self.search_args.update({"productType": product_type})
1031
-
1032
- return parsed_dict