eodag 3.0.0b3__py3-none-any.whl → 3.1.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (94) hide show
  1. eodag/api/core.py +347 -247
  2. eodag/api/product/_assets.py +44 -15
  3. eodag/api/product/_product.py +58 -47
  4. eodag/api/product/drivers/__init__.py +81 -4
  5. eodag/api/product/drivers/base.py +65 -4
  6. eodag/api/product/drivers/generic.py +65 -0
  7. eodag/api/product/drivers/sentinel1.py +97 -0
  8. eodag/api/product/drivers/sentinel2.py +95 -0
  9. eodag/api/product/metadata_mapping.py +129 -93
  10. eodag/api/search_result.py +28 -12
  11. eodag/cli.py +61 -24
  12. eodag/config.py +457 -167
  13. eodag/plugins/apis/base.py +10 -4
  14. eodag/plugins/apis/ecmwf.py +53 -23
  15. eodag/plugins/apis/usgs.py +41 -17
  16. eodag/plugins/authentication/aws_auth.py +30 -18
  17. eodag/plugins/authentication/base.py +14 -3
  18. eodag/plugins/authentication/generic.py +14 -3
  19. eodag/plugins/authentication/header.py +14 -6
  20. eodag/plugins/authentication/keycloak.py +44 -25
  21. eodag/plugins/authentication/oauth.py +18 -4
  22. eodag/plugins/authentication/openid_connect.py +192 -171
  23. eodag/plugins/authentication/qsauth.py +12 -4
  24. eodag/plugins/authentication/sas_auth.py +22 -5
  25. eodag/plugins/authentication/token.py +95 -17
  26. eodag/plugins/authentication/token_exchange.py +19 -19
  27. eodag/plugins/base.py +4 -4
  28. eodag/plugins/crunch/base.py +8 -5
  29. eodag/plugins/crunch/filter_date.py +9 -6
  30. eodag/plugins/crunch/filter_latest_intersect.py +9 -8
  31. eodag/plugins/crunch/filter_latest_tpl_name.py +8 -8
  32. eodag/plugins/crunch/filter_overlap.py +9 -11
  33. eodag/plugins/crunch/filter_property.py +10 -10
  34. eodag/plugins/download/aws.py +181 -105
  35. eodag/plugins/download/base.py +49 -67
  36. eodag/plugins/download/creodias_s3.py +40 -2
  37. eodag/plugins/download/http.py +247 -223
  38. eodag/plugins/download/s3rest.py +29 -28
  39. eodag/plugins/manager.py +176 -41
  40. eodag/plugins/search/__init__.py +6 -5
  41. eodag/plugins/search/base.py +123 -60
  42. eodag/plugins/search/build_search_result.py +1046 -355
  43. eodag/plugins/search/cop_marine.py +132 -39
  44. eodag/plugins/search/creodias_s3.py +19 -68
  45. eodag/plugins/search/csw.py +48 -8
  46. eodag/plugins/search/data_request_search.py +124 -23
  47. eodag/plugins/search/qssearch.py +531 -310
  48. eodag/plugins/search/stac_list_assets.py +85 -0
  49. eodag/plugins/search/static_stac_search.py +23 -24
  50. eodag/resources/ext_product_types.json +1 -1
  51. eodag/resources/product_types.yml +1295 -355
  52. eodag/resources/providers.yml +1819 -3010
  53. eodag/resources/stac.yml +3 -163
  54. eodag/resources/stac_api.yml +2 -2
  55. eodag/resources/user_conf_template.yml +115 -99
  56. eodag/rest/cache.py +2 -2
  57. eodag/rest/config.py +3 -4
  58. eodag/rest/constants.py +0 -1
  59. eodag/rest/core.py +157 -117
  60. eodag/rest/errors.py +181 -0
  61. eodag/rest/server.py +57 -339
  62. eodag/rest/stac.py +133 -581
  63. eodag/rest/types/collections_search.py +3 -3
  64. eodag/rest/types/eodag_search.py +41 -30
  65. eodag/rest/types/queryables.py +42 -32
  66. eodag/rest/types/stac_search.py +15 -16
  67. eodag/rest/utils/__init__.py +14 -21
  68. eodag/rest/utils/cql_evaluate.py +6 -6
  69. eodag/rest/utils/rfc3339.py +2 -2
  70. eodag/types/__init__.py +153 -32
  71. eodag/types/bbox.py +2 -2
  72. eodag/types/download_args.py +4 -4
  73. eodag/types/queryables.py +183 -73
  74. eodag/types/search_args.py +6 -6
  75. eodag/types/whoosh.py +127 -3
  76. eodag/utils/__init__.py +228 -106
  77. eodag/utils/exceptions.py +47 -26
  78. eodag/utils/import_system.py +2 -2
  79. eodag/utils/logging.py +37 -77
  80. eodag/utils/repr.py +65 -6
  81. eodag/utils/requests.py +13 -15
  82. eodag/utils/rest.py +2 -2
  83. eodag/utils/s3.py +231 -0
  84. eodag/utils/stac_reader.py +11 -11
  85. {eodag-3.0.0b3.dist-info → eodag-3.1.0.dist-info}/METADATA +81 -81
  86. eodag-3.1.0.dist-info/RECORD +113 -0
  87. {eodag-3.0.0b3.dist-info → eodag-3.1.0.dist-info}/WHEEL +1 -1
  88. {eodag-3.0.0b3.dist-info → eodag-3.1.0.dist-info}/entry_points.txt +5 -2
  89. eodag/resources/constraints/climate-dt.json +0 -13
  90. eodag/resources/constraints/extremes-dt.json +0 -8
  91. eodag/utils/constraints.py +0 -244
  92. eodag-3.0.0b3.dist-info/RECORD +0 -110
  93. {eodag-3.0.0b3.dist-info → eodag-3.1.0.dist-info}/LICENSE +0 -0
  94. {eodag-3.0.0b3.dist-info → eodag-3.1.0.dist-info}/top_level.txt +0 -0
eodag/rest/stac.py CHANGED
@@ -21,18 +21,19 @@ import logging
21
21
  import os
22
22
  from collections import defaultdict
23
23
  from datetime import datetime, timezone
24
- from typing import TYPE_CHECKING, Any, Dict, List, Optional, Tuple, cast
25
- from urllib.parse import parse_qs, urlencode, urlparse, urlunparse
24
+ from typing import TYPE_CHECKING, Any, Optional
25
+ from urllib.parse import (
26
+ parse_qs,
27
+ quote,
28
+ urlencode,
29
+ urlparse,
30
+ urlsplit,
31
+ urlunparse,
32
+ urlunsplit,
33
+ )
26
34
 
27
- import dateutil.parser
28
35
  import geojson
29
- import shapefile
30
- from dateutil import tz
31
- from dateutil.relativedelta import relativedelta
32
36
  from jsonpath_ng.jsonpath import Child
33
- from shapely.geometry import shape
34
- from shapely.geometry.base import BaseGeometry
35
- from shapely.ops import unary_union
36
37
 
37
38
  from eodag.api.product.metadata_mapping import (
38
39
  DEFAULT_METADATA_MAPPING,
@@ -42,7 +43,6 @@ from eodag.api.product.metadata_mapping import (
42
43
  from eodag.rest.config import Settings
43
44
  from eodag.rest.utils.rfc3339 import str_to_interval
44
45
  from eodag.utils import (
45
- DEFAULT_MISSION_START_DATE,
46
46
  deepcopy,
47
47
  dict_items_recursive_apply,
48
48
  format_dict_items,
@@ -50,14 +50,12 @@ from eodag.utils import (
50
50
  jsonpath_parse_dict_items,
51
51
  string_to_jsonpath,
52
52
  update_nested_dict,
53
- urljoin,
54
53
  )
55
54
  from eodag.utils.exceptions import (
56
55
  NoMatchingProductType,
57
56
  NotAvailableError,
58
57
  RequestError,
59
58
  TimeOutError,
60
- ValidationError,
61
59
  )
62
60
  from eodag.utils.requests import fetch_json
63
61
 
@@ -69,8 +67,6 @@ if TYPE_CHECKING:
69
67
 
70
68
  logger = logging.getLogger("eodag.rest.stac")
71
69
 
72
- STAC_CATALOGS_PREFIX = "catalogs"
73
-
74
70
  # fields not to put in item properties
75
71
  COLLECTION_PROPERTIES = [
76
72
  "abstract",
@@ -86,6 +82,8 @@ COLLECTION_PROPERTIES = [
86
82
  "missionEndDate",
87
83
  "keywords",
88
84
  "stacCollection",
85
+ "alias",
86
+ "productType",
89
87
  ]
90
88
  IGNORED_ITEM_PROPERTIES = [
91
89
  "_id",
@@ -99,9 +97,17 @@ IGNORED_ITEM_PROPERTIES = [
99
97
  "qs",
100
98
  "defaultGeometry",
101
99
  "_date",
100
+ "productType",
102
101
  ]
103
102
 
104
103
 
104
+ def _quote_url_path(url: str) -> str:
105
+ parsed = urlsplit(url)
106
+ path = quote(parsed.path)
107
+ components = (parsed.scheme, parsed.netloc, path, parsed.query, parsed.fragment)
108
+ return urlunsplit(components)
109
+
110
+
105
111
  class StacCommon:
106
112
  """Stac common object
107
113
 
@@ -115,7 +121,7 @@ class StacCommon:
115
121
  def __init__(
116
122
  self,
117
123
  url: str,
118
- stac_config: Dict[str, Any],
124
+ stac_config: dict[str, Any],
119
125
  provider: Optional[str],
120
126
  eodag_api: EODataAccessGateway,
121
127
  root: str = "/",
@@ -126,9 +132,9 @@ class StacCommon:
126
132
  self.eodag_api = eodag_api
127
133
  self.root = root.rstrip("/") if len(root) > 1 else root
128
134
 
129
- self.data: Dict[str, Any] = {}
135
+ self.data: dict[str, Any] = {}
130
136
 
131
- def update_data(self, data: Dict[str, Any]) -> None:
137
+ def update_data(self, data: dict[str, Any]) -> None:
132
138
  """Updates data using given input STAC dict data
133
139
 
134
140
  :param data: Catalog data (parsed STAC dict)
@@ -162,15 +168,15 @@ class StacCommon:
162
168
 
163
169
  @staticmethod
164
170
  def get_stac_extension(
165
- url: str, stac_config: Dict[str, Any], extension: str, **kwargs: Any
166
- ) -> Dict[str, str]:
171
+ url: str, stac_config: dict[str, Any], extension: str, **kwargs: Any
172
+ ) -> dict[str, str]:
167
173
  """Parse STAC extension from config and return as dict
168
174
 
169
175
  :param url: Requested URL
170
176
  :param stac_config: STAC configuration from stac.yml conf file
171
177
  :param extension: Extension name
172
178
  :param kwargs: Additional variables needed for parsing extension
173
- :returns: STAC extension as dictionnary
179
+ :returns: STAC extension as dictionary
174
180
  """
175
181
  extension_model = deepcopy(stac_config).get("extensions", {}).get(extension, {})
176
182
 
@@ -182,7 +188,7 @@ class StacCommon:
182
188
  }
183
189
  return format_dict_items(extension_model, **format_args)
184
190
 
185
- def get_provider_dict(self, provider: str) -> Dict[str, Any]:
191
+ def get_provider_dict(self, provider: str) -> dict[str, Any]:
186
192
  """Generate STAC provider dict"""
187
193
  provider_config = next(
188
194
  p
@@ -211,7 +217,7 @@ class StacItem(StacCommon):
211
217
  def __init__(
212
218
  self,
213
219
  url: str,
214
- stac_config: Dict[str, Any],
220
+ stac_config: dict[str, Any],
215
221
  provider: Optional[str],
216
222
  eodag_api: EODataAccessGateway,
217
223
  root: str = "/",
@@ -225,8 +231,8 @@ class StacItem(StacCommon):
225
231
  )
226
232
 
227
233
  def __get_item_list(
228
- self, search_results: SearchResult, catalog: Dict[str, Any]
229
- ) -> List[Dict[str, Any]]:
234
+ self, search_results: SearchResult, catalog: dict[str, Any]
235
+ ) -> list[dict[str, Any]]:
230
236
  """Build STAC items list from EODAG search results
231
237
 
232
238
  :param search_results: EODAG search results
@@ -241,7 +247,7 @@ class StacItem(StacCommon):
241
247
  )
242
248
 
243
249
  # check if some items need to be converted
244
- need_conversion: Dict[str, Any] = {}
250
+ need_conversion: dict[str, Any] = {}
245
251
  for k, v in item_model["properties"].items():
246
252
  if isinstance(v, str):
247
253
  conversion, item_model["properties"][k] = get_metadata_path(
@@ -261,11 +267,11 @@ class StacItem(StacCommon):
261
267
  ]
262
268
  ignored_props = COLLECTION_PROPERTIES + item_props + IGNORED_ITEM_PROPERTIES
263
269
 
264
- item_list: List[Dict[str, Any]] = []
270
+ item_list: list[dict[str, Any]] = []
265
271
  for product in search_results:
266
272
  product_dict = deepcopy(product.__dict__)
267
273
 
268
- product_item: Dict[str, Any] = jsonpath_parse_dict_items(
274
+ product_item: dict[str, Any] = jsonpath_parse_dict_items(
269
275
  item_model,
270
276
  {
271
277
  "product": product_dict,
@@ -342,6 +348,10 @@ class StacItem(StacCommon):
342
348
  # remove empty properties
343
349
  product_item = self.__filter_item_properties_values(product_item)
344
350
 
351
+ # quote invalid characters in links
352
+ for link in product_item["links"]:
353
+ link["href"] = _quote_url_path(link["href"])
354
+
345
355
  # update item link with datacube query-string
346
356
  if _dc_qs or self.provider:
347
357
  url_parts = urlparse(str(product_item["links"][0]["href"]))
@@ -363,10 +373,10 @@ class StacItem(StacCommon):
363
373
  product: EOProduct,
364
374
  downloadlink_href: str,
365
375
  without_arg_url: str,
366
- query_dict: Optional[Dict[str, Any]] = None,
376
+ query_dict: Optional[dict[str, Any]] = None,
367
377
  _dc_qs: Optional[str] = None,
368
- ) -> Dict[str, Any]:
369
- assets: Dict[str, Any] = {}
378
+ ) -> dict[str, Any]:
379
+ assets: dict[str, Any] = {}
370
380
  settings = Settings.from_environment()
371
381
 
372
382
  if _dc_qs:
@@ -378,9 +388,12 @@ class StacItem(StacCommon):
378
388
  origin_href = product.remote_location
379
389
 
380
390
  # update download link with up-to-date query-args
391
+ quoted_href = _quote_url_path(
392
+ downloadlink_href
393
+ ) # quote invalid characters in url
381
394
  assets["downloadLink"] = {
382
395
  "title": "Download link",
383
- "href": downloadlink_href,
396
+ "href": quoted_href,
384
397
  "type": "application/zip",
385
398
  }
386
399
 
@@ -424,6 +437,7 @@ class StacItem(StacCommon):
424
437
  assets[asset_key]["type"] = asset_type
425
438
  if origin := assets[asset_key].get("alternate", {}).get("origin"):
426
439
  origin["type"] = asset_type
440
+ asset_value["href"] = _quote_url_path(asset_value["href"])
427
441
 
428
442
  if thumbnail_url := product.properties.get(
429
443
  "quicklook", product.properties.get("thumbnail", None)
@@ -441,14 +455,14 @@ class StacItem(StacCommon):
441
455
  self,
442
456
  search_results: SearchResult,
443
457
  total: int,
444
- catalog: Dict[str, Any],
445
- next_link: Optional[Dict[str, Any]],
446
- ) -> Dict[str, Any]:
458
+ catalog: dict[str, Any],
459
+ next_link: Optional[dict[str, Any]],
460
+ ) -> dict[str, Any]:
447
461
  """Build STAC items from EODAG search results
448
462
 
449
463
  :param search_results: EODAG search results
450
464
  :param catalog: STAC catalog dict used for parsing item metadata
451
- :returns: Items dictionnary
465
+ :returns: Items dictionary
452
466
  """
453
467
  items_model = deepcopy(self.stac_config["items"])
454
468
 
@@ -492,8 +506,8 @@ class StacItem(StacCommon):
492
506
  return self.data
493
507
 
494
508
  def __filter_item_model_properties(
495
- self, item_model: Dict[str, Any], product_type: str
496
- ) -> Dict[str, Any]:
509
+ self, item_model: dict[str, Any], product_type: str
510
+ ) -> dict[str, Any]:
497
511
  """Filter item model depending on product type metadata and its extensions.
498
512
  Removes not needed parameters, and adds supplementary ones as
499
513
  part of oseo extension.
@@ -557,13 +571,13 @@ class StacItem(StacCommon):
557
571
 
558
572
  return result_item_model
559
573
 
560
- def __filter_item_properties_values(self, item: Dict[str, Any]) -> Dict[str, Any]:
574
+ def __filter_item_properties_values(self, item: dict[str, Any]) -> dict[str, Any]:
561
575
  """Removes empty properties, unused extensions, and add missing extensions
562
576
 
563
577
  :param item: STAC item data
564
578
  :returns: Filtered item model
565
579
  """
566
- all_extensions_dict: Dict[str, str] = deepcopy(
580
+ all_extensions_dict: dict[str, str] = deepcopy(
567
581
  self.stac_config["stac_extensions"]
568
582
  )
569
583
  # parse f-strings with root
@@ -588,7 +602,7 @@ class StacItem(StacCommon):
588
602
 
589
603
  return item
590
604
 
591
- def get_stac_item_from_product(self, product: EOProduct) -> Dict[str, Any]:
605
+ def get_stac_item_from_product(self, product: EOProduct) -> dict[str, Any]:
592
606
  """Build STAC item from EODAG product
593
607
 
594
608
  :param product: EODAG product
@@ -606,7 +620,7 @@ class StacItem(StacCommon):
606
620
  root=self.root,
607
621
  provider=self.provider,
608
622
  eodag_api=self.eodag_api,
609
- catalogs=[product_type],
623
+ collection=product_type,
610
624
  )
611
625
 
612
626
  product_dict = deepcopy(product.__dict__)
@@ -648,7 +662,7 @@ class StacCollection(StacCommon):
648
662
  """
649
663
 
650
664
  # External STAC collections
651
- ext_stac_collections: Dict[str, Dict[str, Any]] = dict()
665
+ ext_stac_collections: dict[str, dict[str, Any]] = dict()
652
666
 
653
667
  @classmethod
654
668
  def fetch_external_stac_collections(cls, eodag_api: EODataAccessGateway) -> None:
@@ -677,7 +691,7 @@ class StacCollection(StacCommon):
677
691
  def __init__(
678
692
  self,
679
693
  url: str,
680
- stac_config: Dict[str, Any],
694
+ stac_config: dict[str, Any],
681
695
  provider: Optional[str],
682
696
  eodag_api: EODataAccessGateway,
683
697
  root: str = "/",
@@ -690,7 +704,7 @@ class StacCollection(StacCommon):
690
704
  root=root,
691
705
  )
692
706
 
693
- def __list_product_type_providers(self, product_type: Dict[str, Any]) -> List[str]:
707
+ def __list_product_type_providers(self, product_type: dict[str, Any]) -> list[str]:
694
708
  """Retrieve a list of providers for a given product type.
695
709
 
696
710
  :param product_type: Dictionary containing information about the product type.
@@ -707,8 +721,8 @@ class StacCollection(StacCommon):
707
721
  ]
708
722
 
709
723
  def __generate_stac_collection(
710
- self, collection_model: Any, product_type: Dict[str, Any]
711
- ) -> Dict[str, Any]:
724
+ self, collection_model: Any, product_type: dict[str, Any]
725
+ ) -> dict[str, Any]:
712
726
  """Generate a STAC collection dictionary for a given product type.
713
727
 
714
728
  :param collection_model: The base model for the STAC collection.
@@ -717,7 +731,7 @@ class StacCollection(StacCommon):
717
731
  """
718
732
  providers = self.__list_product_type_providers(product_type)
719
733
 
720
- providers_dict: Dict[str, Dict[str, Any]] = {}
734
+ providers_dict: dict[str, dict[str, Any]] = {}
721
735
  for provider in providers:
722
736
  p_dict = self.get_provider_dict(provider)
723
737
  providers_dict.setdefault(p_dict["name"], p_dict)
@@ -744,25 +758,34 @@ class StacCollection(StacCommon):
744
758
  ]
745
759
  ext_stac_collection["links"].append(link)
746
760
 
761
+ # merge "summaries"
762
+ ext_stac_collection["summaries"] = {
763
+ k: v
764
+ for k, v in {
765
+ **ext_stac_collection.get("summaries", {}),
766
+ **product_type_collection["summaries"],
767
+ }.items()
768
+ if v and any(v)
769
+ }
770
+
747
771
  # merge "keywords" lists
748
- if "keywords" in ext_stac_collection:
749
- try:
750
- ext_stac_collection["keywords"] = [
751
- k
752
- for k in set(
753
- ext_stac_collection["keywords"]
754
- + product_type_collection["keywords"]
755
- )
756
- if k is not None
757
- ]
758
- except TypeError as e:
759
- logger.warning(
760
- f"Could not merge keywords from external collection for {product_type['ID']}: {str(e)}"
761
- )
762
- logger.debug(
763
- f"External collection keywords: {str(ext_stac_collection['keywords'])}, ",
764
- f"Product type keywords: {str(product_type_collection['keywords'])}",
772
+ try:
773
+ ext_stac_collection["keywords"] = [
774
+ k
775
+ for k in set(
776
+ ext_stac_collection.get("keywords", [])
777
+ + product_type_collection["keywords"]
765
778
  )
779
+ if k is not None
780
+ ]
781
+ except TypeError as e:
782
+ logger.warning(
783
+ f"Could not merge keywords from external collection for {product_type['ID']}: {str(e)}"
784
+ )
785
+ logger.debug(
786
+ f"External collection keywords: {str(ext_stac_collection.get('keywords'))}, ",
787
+ f"Product type keywords: {str(product_type_collection['keywords'])}",
788
+ )
766
789
 
767
790
  # merge providers
768
791
  if "providers" in ext_stac_collection:
@@ -795,7 +818,7 @@ class StacCollection(StacCommon):
795
818
  instrument: Optional[str] = None,
796
819
  constellation: Optional[str] = None,
797
820
  datetime: Optional[str] = None,
798
- ) -> List[Dict[str, Any]]:
821
+ ) -> list[dict[str, Any]]:
799
822
  """Build STAC collections list
800
823
 
801
824
  :param filters: (optional) Additional filters for collections search
@@ -830,7 +853,7 @@ class StacCollection(StacCommon):
830
853
  product_types = all_pt
831
854
 
832
855
  # list product types with all metadata using guessed ids
833
- collection_list: List[Dict[str, Any]] = []
856
+ collection_list: list[dict[str, Any]] = []
834
857
  for product_type in product_types:
835
858
  stac_collection = self.__generate_stac_collection(
836
859
  collection_model, product_type
@@ -848,17 +871,17 @@ class StacCatalog(StacCommon):
848
871
  :param provider: Chosen provider
849
872
  :param eodag_api: EODAG python API instance
850
873
  :param root: (optional) API root
851
- :param catalogs: (optional) Catalogs list
874
+ :param collection: (optional) product type id
852
875
  """
853
876
 
854
877
  def __init__(
855
878
  self,
856
879
  url: str,
857
- stac_config: Dict[str, Any],
880
+ stac_config: dict[str, Any],
858
881
  provider: Optional[str],
859
882
  eodag_api: EODataAccessGateway,
860
883
  root: str = "/",
861
- catalogs: Optional[List[str]] = None,
884
+ collection: Optional[str] = None,
862
885
  ) -> None:
863
886
  super(StacCatalog, self).__init__(
864
887
  url=url,
@@ -870,8 +893,8 @@ class StacCatalog(StacCommon):
870
893
  self.data = {}
871
894
 
872
895
  self.shp_location_config = eodag_api.locations_config
873
- self.search_args: Dict[str, Any] = {}
874
- self.children: List[Dict[str, Any]] = []
896
+ self.search_args: dict[str, Any] = {}
897
+ self.children: list[dict[str, Any]] = []
875
898
 
876
899
  self.catalog_config = deepcopy(stac_config["catalog"])
877
900
 
@@ -885,9 +908,9 @@ class StacCatalog(StacCommon):
885
908
  self.data["links"] += self.children
886
909
 
887
910
  # build catalog
888
- self.__build_stac_catalog(catalogs)
911
+ self.__build_stac_catalog(collection)
889
912
 
890
- def __update_data_from_catalog_config(self, catalog_config: Dict[str, Any]) -> bool:
913
+ def __update_data_from_catalog_config(self, catalog_config: dict[str, Any]) -> bool:
891
914
  """Updates configuration and data using given input catalog config
892
915
 
893
916
  :param catalog_config: Catalog config, from yml stac_config[catalogs]
@@ -910,21 +933,36 @@ class StacCatalog(StacCommon):
910
933
 
911
934
  return True
912
935
 
913
- def set_children(self, children: Optional[List[Dict[str, Any]]] = None) -> bool:
914
- """Set catalog children / links
936
+ def __build_stac_catalog(self, collection: Optional[str] = None) -> StacCatalog:
937
+ """Build nested catalog from catalag list
915
938
 
916
- :param children: (optional) Children list
939
+ :param collection: (optional) product type id
940
+ :returns: This catalog obj
917
941
  """
918
- self.children = children or []
919
- self.data["links"] = [
920
- link for link in self.data["links"] if link["rel"] != "child"
921
- ]
922
- self.data["links"] += self.children
923
- return True
942
+ settings = Settings.from_environment()
943
+
944
+ if not collection:
945
+ # Build root catalog combined with landing page
946
+ self.__update_data_from_catalog_config(
947
+ {
948
+ "model": {
949
+ **deepcopy(self.stac_config["landing_page"]),
950
+ **{
951
+ "provider": self.provider,
952
+ "id": settings.stac_api_landing_id,
953
+ "title": settings.stac_api_title,
954
+ "description": settings.stac_api_description,
955
+ },
956
+ }
957
+ }
958
+ )
959
+ else:
960
+ self.set_stac_product_type_by_id(collection)
961
+ return self
924
962
 
925
963
  def set_stac_product_type_by_id(
926
964
  self, product_type: str, **_: Any
927
- ) -> Dict[str, Any]:
965
+ ) -> dict[str, Any]:
928
966
  """Updates catalog with given product_type
929
967
 
930
968
  :param product_type: Product type
@@ -940,13 +978,23 @@ class StacCatalog(StacCommon):
940
978
  if not collections:
941
979
  raise NotAvailableError(f"Collection {product_type} does not exist.")
942
980
 
943
- cat_model = deepcopy(self.stac_config["catalogs"]["product_type"]["model"])
981
+ cat_model = {
982
+ "id": "{collection[id]}",
983
+ "title": "{collection[title]}",
984
+ "description": "{collection[description]}",
985
+ "extent": "{collection[extent]}",
986
+ "crs": "http://www.opengis.net/def/crs/OGC/1.3/CRS84",
987
+ "keywords": "{collection[keywords]}",
988
+ "license": "{collection[license]}",
989
+ "providers": "{collection[providers]}",
990
+ "summaries": "{collection[summaries]}",
991
+ }
944
992
  # parse f-strings
945
993
  format_args = deepcopy(self.stac_config)
946
994
  format_args["catalog"] = defaultdict(str, **self.data)
947
995
  format_args["collection"] = collections[0]
948
996
  try:
949
- parsed_dict: Dict[str, Any] = format_dict_items(cat_model, **format_args)
997
+ parsed_dict: dict[str, Any] = format_dict_items(cat_model, **format_args)
950
998
  except Exception:
951
999
  logger.error("Could not format product_type catalog")
952
1000
  raise
@@ -957,499 +1005,3 @@ class StacCatalog(StacCommon):
957
1005
  self.search_args.update({"productType": product_type})
958
1006
 
959
1007
  return parsed_dict
960
-
961
- # get / set dates filters -------------------------------------------------
962
-
963
- def get_stac_years_list(self, **_: Any) -> List[int]:
964
- """Get catalog available years list
965
-
966
- :returns: Years list
967
- """
968
- extent_date_min, extent_date_max = self.get_datetime_extent()
969
-
970
- return list(range(extent_date_min.year, extent_date_max.year + 1))
971
-
972
- def get_stac_months_list(self, **_: Any) -> List[int]:
973
- """Get catalog available months list
974
-
975
- :returns: Months list
976
- """
977
- extent_date_min, extent_date_max = self.get_datetime_extent()
978
-
979
- return list(
980
- range(
981
- extent_date_min.month,
982
- (extent_date_max - relativedelta(days=1)).month + 1,
983
- )
984
- )
985
-
986
- def get_stac_days_list(self, **_: Any) -> List[int]:
987
- """Get catalog available days list
988
-
989
- :returns: Days list
990
- """
991
- extent_date_min, extent_date_max = self.get_datetime_extent()
992
-
993
- return list(
994
- range(
995
- extent_date_min.day, (extent_date_max - relativedelta(days=1)).day + 1
996
- )
997
- )
998
-
999
- def set_stac_year_by_id(self, year: str, **_: Any) -> Dict[str, Any]:
1000
- """Updates and returns catalog with given year
1001
-
1002
- :param year: Year number
1003
- :returns: Updated catalog
1004
- """
1005
- extent_date_min, extent_date_max = self.get_datetime_extent()
1006
-
1007
- datetime_min = max(
1008
- [extent_date_min, dateutil.parser.parse(f"{year}-01-01T00:00:00Z")]
1009
- )
1010
- datetime_max = min(
1011
- [
1012
- extent_date_max,
1013
- dateutil.parser.parse(f"{year}-01-01T00:00:00Z")
1014
- + relativedelta(years=1),
1015
- ]
1016
- )
1017
-
1018
- catalog_model = deepcopy(self.stac_config["catalogs"]["year"]["model"])
1019
-
1020
- parsed_dict = self.set_stac_date(datetime_min, datetime_max, catalog_model)
1021
-
1022
- return parsed_dict
1023
-
1024
- def set_stac_month_by_id(self, month: str, **_: Any) -> Dict[str, Any]:
1025
- """Updates and returns catalog with given month
1026
-
1027
- :param month: Month number
1028
- :returns: Updated catalog
1029
- """
1030
- extent_date_min, extent_date_max = self.get_datetime_extent()
1031
- year = extent_date_min.year
1032
-
1033
- datetime_min = max(
1034
- [
1035
- extent_date_min,
1036
- dateutil.parser.parse(f"{year}-{month}-01T00:00:00Z"),
1037
- ]
1038
- )
1039
- datetime_max = min(
1040
- [
1041
- extent_date_max,
1042
- dateutil.parser.parse(f"{year}-{month}-01T00:00:00Z")
1043
- + relativedelta(months=1),
1044
- ]
1045
- )
1046
-
1047
- catalog_model = deepcopy(self.stac_config["catalogs"]["month"]["model"])
1048
-
1049
- parsed_dict = self.set_stac_date(datetime_min, datetime_max, catalog_model)
1050
-
1051
- return parsed_dict
1052
-
1053
- def set_stac_day_by_id(self, day: str, **_: Any) -> Dict[str, Any]:
1054
- """Updates and returns catalog with given day
1055
-
1056
- :param day: Day number
1057
- :returns: Updated catalog
1058
- """
1059
- extent_date_min, extent_date_max = self.get_datetime_extent()
1060
- year = extent_date_min.year
1061
- month = extent_date_min.month
1062
-
1063
- datetime_min = max(
1064
- [
1065
- extent_date_min,
1066
- dateutil.parser.parse(f"{year}-{month}-{day}T00:00:00Z"),
1067
- ]
1068
- )
1069
- datetime_max = min(
1070
- [
1071
- extent_date_max,
1072
- dateutil.parser.parse(f"{year}-{month}-{day}T00:00:00Z")
1073
- + relativedelta(days=1),
1074
- ]
1075
- )
1076
-
1077
- catalog_model = deepcopy(self.stac_config["catalogs"]["day"]["model"])
1078
-
1079
- parsed_dict = self.set_stac_date(datetime_min, datetime_max, catalog_model)
1080
-
1081
- return parsed_dict
1082
-
1083
- def get_datetime_extent(self) -> Tuple[datetime, datetime]:
1084
- """Returns catalog temporal extent as datetime objs
1085
-
1086
- :returns: Start & stop dates
1087
- """
1088
- extent_date_min = dateutil.parser.parse(DEFAULT_MISSION_START_DATE).replace(
1089
- tzinfo=tz.UTC
1090
- )
1091
- extent_date_max = datetime.now(timezone.utc).replace(tzinfo=tz.UTC)
1092
- for interval in self.data["extent"]["temporal"]["interval"]:
1093
- extent_date_min_str, extent_date_max_str = interval
1094
- # date min
1095
- if extent_date_min_str:
1096
- extent_date_min = max(
1097
- extent_date_min, dateutil.parser.parse(extent_date_min_str)
1098
- )
1099
- # date max
1100
- if extent_date_max_str:
1101
- extent_date_max = min(
1102
- extent_date_max, dateutil.parser.parse(extent_date_max_str)
1103
- )
1104
-
1105
- return (
1106
- extent_date_min.replace(tzinfo=tz.UTC),
1107
- extent_date_max.replace(tzinfo=tz.UTC),
1108
- )
1109
-
1110
- def set_stac_date(
1111
- self,
1112
- datetime_min: datetime,
1113
- datetime_max: datetime,
1114
- catalog_model: Dict[str, Any],
1115
- ):
1116
- """Updates catalog data using given dates
1117
-
1118
- :param datetime_min: Date min of interval
1119
- :param datetime_max: Date max of interval
1120
- :param catalog_model: Catalog model to use, from yml stac_config[catalogs]
1121
- :returns: Updated catalog
1122
- """
1123
- # parse f-strings
1124
- format_args = deepcopy(self.stac_config)
1125
- format_args["catalog"] = defaultdict(str, **self.data)
1126
- format_args["date"] = defaultdict(
1127
- str,
1128
- {
1129
- "year": datetime_min.year,
1130
- "month": datetime_min.month,
1131
- "day": datetime_min.day,
1132
- "min": datetime_min.isoformat().replace("+00:00", "Z"),
1133
- "max": datetime_max.isoformat().replace("+00:00", "Z"),
1134
- },
1135
- )
1136
- parsed_dict: Dict[str, Any] = format_dict_items(catalog_model, **format_args)
1137
-
1138
- self.update_data(parsed_dict)
1139
-
1140
- # update search args
1141
- self.search_args.update(
1142
- {
1143
- "start": datetime_min.isoformat().replace("+00:00", "Z"),
1144
- "end": datetime_max.isoformat().replace("+00:00", "Z"),
1145
- }
1146
- )
1147
- return parsed_dict
1148
-
1149
- # get / set cloud_cover filter --------------------------------------------
1150
-
1151
- def get_stac_cloud_covers_list(self, **_: Any) -> List[int]:
1152
- """Get cloud_cover list
1153
-
1154
- :returns: cloud_cover list
1155
- """
1156
- return list(range(0, 101, 10))
1157
-
1158
- def set_stac_cloud_cover_by_id(self, cloud_cover: str, **_: Any) -> Dict[str, Any]:
1159
- """Updates and returns catalog with given max cloud_cover
1160
-
1161
- :param cloud_cover: Cloud_cover number
1162
- :returns: Updated catalog
1163
- """
1164
- cat_model = deepcopy(self.stac_config["catalogs"]["cloud_cover"]["model"])
1165
- # parse f-strings
1166
- format_args = deepcopy(self.stac_config)
1167
- format_args["catalog"] = defaultdict(str, **self.data)
1168
- format_args["cloud_cover"] = cloud_cover
1169
- parsed_dict: Dict[str, Any] = format_dict_items(cat_model, **format_args)
1170
-
1171
- self.update_data(parsed_dict)
1172
-
1173
- # update search args
1174
- self.search_args.update({"cloudCover": cloud_cover})
1175
-
1176
- return parsed_dict
1177
-
1178
- # get / set locations filter ----------------------------------------------
1179
-
1180
- def get_stac_location_list(self, catalog_name: str) -> List[str]:
1181
- """Get locations list using stac_conf & locations_config
1182
-
1183
- :param catalog_name: Catalog/location name
1184
- :returns: Locations list
1185
- """
1186
-
1187
- if catalog_name not in self.stac_config["catalogs"]:
1188
- logger.warning("no entry found for %s in location_config", catalog_name)
1189
- return []
1190
- location_config = self.stac_config["catalogs"][catalog_name]
1191
-
1192
- for k in ["path", "attr"]:
1193
- if k not in location_config.keys():
1194
- logger.warning(
1195
- "no %s key found for %s in location_config", k, catalog_name
1196
- )
1197
- return []
1198
- path = location_config["path"]
1199
- attr = location_config["attr"]
1200
-
1201
- with shapefile.Reader(path) as shp:
1202
- countries_list: List[str] = [rec[attr] for rec in shp.records()] # type: ignore
1203
-
1204
- # remove duplicates
1205
- countries_list = list(set(countries_list))
1206
-
1207
- countries_list.sort()
1208
-
1209
- return countries_list
1210
-
1211
- def set_stac_location_by_id(
1212
- self, location: str, catalog_name: str
1213
- ) -> Dict[str, Any]:
1214
- """Updates and returns catalog with given location
1215
-
1216
- :param location: Feature attribute value for shp filtering
1217
- :param catalog_name: Catalog/location name
1218
- :returns: Updated catalog
1219
- """
1220
- location_list_cat_key = catalog_name + "_list"
1221
-
1222
- if location_list_cat_key not in self.stac_config["catalogs"]:
1223
- logger.warning(
1224
- "no entry found for %s's list in location_config", catalog_name
1225
- )
1226
- return {}
1227
- location_config = self.stac_config["catalogs"][location_list_cat_key]
1228
-
1229
- for k in ["path", "attr"]:
1230
- if k not in location_config.keys():
1231
- logger.warning(
1232
- "no %s key found for %s's list in location_config", k, catalog_name
1233
- )
1234
- return {}
1235
- path = location_config["path"]
1236
- attr = location_config["attr"]
1237
-
1238
- with shapefile.Reader(path) as shp:
1239
- geom_hits = [
1240
- shape(shaperec.shape)
1241
- for shaperec in shp.shapeRecords()
1242
- if shaperec.record.as_dict().get(attr, None) == location
1243
- ]
1244
-
1245
- if not geom_hits:
1246
- logger.warning(
1247
- "no feature found in %s matching %s=%s", path, attr, location
1248
- )
1249
- return {}
1250
-
1251
- geom = cast(BaseGeometry, unary_union(geom_hits))
1252
-
1253
- cat_model = deepcopy(self.stac_config["catalogs"]["country"]["model"])
1254
- # parse f-strings
1255
- format_args = deepcopy(self.stac_config)
1256
- format_args["catalog"] = defaultdict(str, **self.data)
1257
- format_args["feature"] = defaultdict(str, {"geometry": geom, "id": location})
1258
- parsed_dict: Dict[str, Any] = format_dict_items(cat_model, **format_args)
1259
-
1260
- self.update_data(parsed_dict)
1261
-
1262
- # update search args
1263
- self.search_args.update({"geom": geom})
1264
-
1265
- return parsed_dict
1266
-
1267
- def build_locations_config(self) -> Dict[str, str]:
1268
- """Build locations config from stac_conf[locations_catalogs] & eodag_api.locations_config
1269
-
1270
- :returns: Locations configuration dict
1271
- """
1272
- user_config_locations_list = self.eodag_api.locations_config
1273
-
1274
- locations_config_model = deepcopy(self.stac_config["locations_catalogs"])
1275
-
1276
- locations_config: Dict[str, str] = {}
1277
- for loc in user_config_locations_list:
1278
- # parse jsonpath
1279
- parsed = jsonpath_parse_dict_items(
1280
- locations_config_model, {"shp_location": loc}
1281
- )
1282
-
1283
- # set default child/parent for this location
1284
- parsed["location"]["parent_key"] = f"{loc['name']}_list"
1285
-
1286
- locations_config[f"{loc['name']}_list"] = parsed["locations_list"]
1287
- locations_config[loc["name"]] = parsed["location"]
1288
-
1289
- return locations_config
1290
-
1291
- def __build_stac_catalog(self, catalogs: Optional[List[str]] = None) -> StacCatalog:
1292
- """Build nested catalog from catalag list
1293
-
1294
- :param catalogs: (optional) Catalogs list
1295
- :returns: This catalog obj
1296
- """
1297
- settings = Settings.from_environment()
1298
-
1299
- # update conf with user shp locations
1300
- locations_config = self.build_locations_config()
1301
-
1302
- self.stac_config["catalogs"] = {
1303
- **deepcopy(self.stac_config["catalogs"]),
1304
- **locations_config,
1305
- }
1306
-
1307
- if not catalogs:
1308
- # Build root catalog combined with landing page
1309
- self.__update_data_from_catalog_config(
1310
- {
1311
- "model": {
1312
- **deepcopy(self.stac_config["landing_page"]),
1313
- **{
1314
- "provider": self.provider,
1315
- "id": settings.stac_api_landing_id,
1316
- "title": settings.stac_api_title,
1317
- "description": settings.stac_api_description,
1318
- },
1319
- }
1320
- }
1321
- )
1322
-
1323
- # build children : product_types
1324
- product_types_list = [
1325
- pt
1326
- for pt in self.eodag_api.list_product_types(
1327
- provider=self.provider, fetch_providers=False
1328
- )
1329
- ]
1330
- self.set_children(
1331
- [
1332
- {
1333
- "rel": "child",
1334
- "href": urljoin(
1335
- self.url, f"{STAC_CATALOGS_PREFIX}/{product_type['ID']}"
1336
- ),
1337
- "title": product_type["title"],
1338
- }
1339
- for product_type in product_types_list
1340
- ]
1341
- )
1342
- return self
1343
-
1344
- # use product_types_list as base for building nested catalogs
1345
- self.__update_data_from_catalog_config(
1346
- deepcopy(self.stac_config["catalogs"]["product_types_list"])
1347
- )
1348
-
1349
- for idx, cat in enumerate(catalogs):
1350
- if idx % 2 == 0:
1351
- # even: cat is a filtering value ----------------------------------
1352
- cat_data_name = self.catalog_config["child_key"]
1353
- cat_data_value = cat
1354
-
1355
- # update data
1356
- cat_data_name_dict = self.stac_config["catalogs"][cat_data_name]
1357
- set_data_method_name = (
1358
- f"set_stac_{cat_data_name}_by_id"
1359
- if "catalog_type" not in cat_data_name_dict.keys()
1360
- else f"set_stac_{cat_data_name_dict['catalog_type']}_by_id"
1361
- )
1362
- set_data_method = getattr(self, set_data_method_name)
1363
- set_data_method(cat_data_value, catalog_name=cat_data_name)
1364
-
1365
- if idx == len(catalogs) - 1:
1366
- # build children : remaining filtering keys
1367
- remaining_catalogs_list = [
1368
- c
1369
- for c in self.stac_config["catalogs"].keys()
1370
- # keep filters not used yet AND
1371
- if self.stac_config["catalogs"][c]["model"]["id"]
1372
- not in catalogs
1373
- and (
1374
- # filters with no parent_key constraint (no key, or key=None) OR
1375
- "parent_key" not in self.stac_config["catalogs"][c]
1376
- or not self.stac_config["catalogs"][c]["parent_key"]
1377
- # filters matching parent_key constraint
1378
- or self.stac_config["catalogs"][c]["parent_key"]
1379
- == cat_data_name
1380
- )
1381
- # AND filters that match parent attr constraint (locations)
1382
- and (
1383
- "parent" not in self.stac_config["catalogs"][c]
1384
- or not self.stac_config["catalogs"][c]["parent"]["key"]
1385
- or (
1386
- self.stac_config["catalogs"][c]["parent"]["key"]
1387
- == cat_data_name
1388
- and self.stac_config["catalogs"][c]["parent"]["attr"]
1389
- == cat_data_value
1390
- )
1391
- )
1392
- ]
1393
-
1394
- self.set_children(
1395
- [
1396
- {
1397
- "rel": "child",
1398
- "href": self.url
1399
- + "/"
1400
- + self.stac_config["catalogs"][c]["model"]["id"],
1401
- "title": str(
1402
- self.stac_config["catalogs"][c]["model"]["id"]
1403
- ),
1404
- }
1405
- for c in remaining_catalogs_list
1406
- ]
1407
- + [
1408
- {
1409
- "rel": "items",
1410
- "href": self.url + "/items",
1411
- "title": "items",
1412
- }
1413
- ]
1414
- )
1415
-
1416
- else:
1417
- # odd: cat is a filtering key -------------------------------------
1418
- try:
1419
- cat_key = [
1420
- c
1421
- for c in self.stac_config["catalogs"].keys()
1422
- if self.stac_config["catalogs"][c]["model"]["id"] == cat
1423
- ][0]
1424
- except IndexError as e:
1425
- raise ValidationError(
1426
- f"Bad settings for {cat} in stac_config catalogs"
1427
- ) from e
1428
- cat_config = deepcopy(self.stac_config["catalogs"][cat_key])
1429
- # update data
1430
- self.__update_data_from_catalog_config(cat_config)
1431
-
1432
- # get filtering values list
1433
- get_data_method_name = (
1434
- f"get_stac_{cat_key}"
1435
- if "catalog_type"
1436
- not in self.stac_config["catalogs"][cat_key].keys()
1437
- else f"get_stac_{self.stac_config['catalogs'][cat_key]['catalog_type']}"
1438
- )
1439
- get_data_method = getattr(self, get_data_method_name)
1440
- cat_data_list = get_data_method(catalog_name=cat_key)
1441
-
1442
- if idx == len(catalogs) - 1:
1443
- # filtering values list as children (do not include items)
1444
- self.set_children(
1445
- [
1446
- {
1447
- "rel": "child",
1448
- "href": self.url + "/" + str(filtering_data),
1449
- "title": str(filtering_data),
1450
- }
1451
- for filtering_data in cat_data_list
1452
- ]
1453
- )
1454
-
1455
- return self