eodag 3.0.0b2__py3-none-any.whl → 3.0.1__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (84) hide show
  1. eodag/__init__.py +6 -8
  2. eodag/api/core.py +295 -287
  3. eodag/api/product/__init__.py +10 -4
  4. eodag/api/product/_assets.py +2 -14
  5. eodag/api/product/_product.py +16 -30
  6. eodag/api/product/drivers/__init__.py +7 -2
  7. eodag/api/product/drivers/base.py +0 -3
  8. eodag/api/product/metadata_mapping.py +12 -31
  9. eodag/api/search_result.py +33 -12
  10. eodag/cli.py +35 -19
  11. eodag/config.py +455 -155
  12. eodag/plugins/apis/base.py +13 -7
  13. eodag/plugins/apis/ecmwf.py +16 -7
  14. eodag/plugins/apis/usgs.py +68 -16
  15. eodag/plugins/authentication/aws_auth.py +25 -7
  16. eodag/plugins/authentication/base.py +10 -1
  17. eodag/plugins/authentication/generic.py +14 -3
  18. eodag/plugins/authentication/header.py +12 -4
  19. eodag/plugins/authentication/keycloak.py +41 -22
  20. eodag/plugins/authentication/oauth.py +11 -1
  21. eodag/plugins/authentication/openid_connect.py +183 -167
  22. eodag/plugins/authentication/qsauth.py +12 -4
  23. eodag/plugins/authentication/sas_auth.py +19 -2
  24. eodag/plugins/authentication/token.py +59 -11
  25. eodag/plugins/authentication/token_exchange.py +19 -19
  26. eodag/plugins/crunch/base.py +7 -2
  27. eodag/plugins/crunch/filter_date.py +8 -11
  28. eodag/plugins/crunch/filter_latest_intersect.py +5 -7
  29. eodag/plugins/crunch/filter_latest_tpl_name.py +2 -5
  30. eodag/plugins/crunch/filter_overlap.py +9 -15
  31. eodag/plugins/crunch/filter_property.py +9 -14
  32. eodag/plugins/download/aws.py +84 -99
  33. eodag/plugins/download/base.py +36 -77
  34. eodag/plugins/download/creodias_s3.py +11 -2
  35. eodag/plugins/download/http.py +134 -109
  36. eodag/plugins/download/s3rest.py +37 -43
  37. eodag/plugins/manager.py +173 -41
  38. eodag/plugins/search/__init__.py +9 -9
  39. eodag/plugins/search/base.py +35 -35
  40. eodag/plugins/search/build_search_result.py +55 -64
  41. eodag/plugins/search/cop_marine.py +113 -32
  42. eodag/plugins/search/creodias_s3.py +20 -8
  43. eodag/plugins/search/csw.py +41 -1
  44. eodag/plugins/search/data_request_search.py +119 -14
  45. eodag/plugins/search/qssearch.py +619 -197
  46. eodag/plugins/search/static_stac_search.py +25 -23
  47. eodag/resources/ext_product_types.json +1 -1
  48. eodag/resources/product_types.yml +211 -56
  49. eodag/resources/providers.yml +1762 -1809
  50. eodag/resources/stac.yml +3 -163
  51. eodag/resources/user_conf_template.yml +134 -119
  52. eodag/rest/config.py +1 -2
  53. eodag/rest/constants.py +0 -1
  54. eodag/rest/core.py +70 -92
  55. eodag/rest/errors.py +181 -0
  56. eodag/rest/server.py +24 -330
  57. eodag/rest/stac.py +105 -630
  58. eodag/rest/types/eodag_search.py +17 -15
  59. eodag/rest/types/queryables.py +5 -14
  60. eodag/rest/types/stac_search.py +18 -13
  61. eodag/rest/utils/rfc3339.py +0 -1
  62. eodag/types/__init__.py +24 -6
  63. eodag/types/download_args.py +14 -5
  64. eodag/types/queryables.py +1 -2
  65. eodag/types/search_args.py +10 -11
  66. eodag/types/whoosh.py +0 -2
  67. eodag/utils/__init__.py +97 -136
  68. eodag/utils/constraints.py +0 -8
  69. eodag/utils/exceptions.py +23 -9
  70. eodag/utils/import_system.py +0 -4
  71. eodag/utils/logging.py +37 -80
  72. eodag/utils/notebook.py +4 -4
  73. eodag/utils/requests.py +13 -23
  74. eodag/utils/rest.py +0 -4
  75. eodag/utils/stac_reader.py +3 -15
  76. {eodag-3.0.0b2.dist-info → eodag-3.0.1.dist-info}/METADATA +41 -24
  77. eodag-3.0.1.dist-info/RECORD +109 -0
  78. {eodag-3.0.0b2.dist-info → eodag-3.0.1.dist-info}/WHEEL +1 -1
  79. {eodag-3.0.0b2.dist-info → eodag-3.0.1.dist-info}/entry_points.txt +1 -0
  80. eodag/resources/constraints/climate-dt.json +0 -13
  81. eodag/resources/constraints/extremes-dt.json +0 -8
  82. eodag-3.0.0b2.dist-info/RECORD +0 -110
  83. {eodag-3.0.0b2.dist-info → eodag-3.0.1.dist-info}/LICENSE +0 -0
  84. {eodag-3.0.0b2.dist-info → eodag-3.0.1.dist-info}/top_level.txt +0 -0
eodag/rest/stac.py CHANGED
@@ -21,18 +21,19 @@ import logging
21
21
  import os
22
22
  from collections import defaultdict
23
23
  from datetime import datetime, timezone
24
- from typing import TYPE_CHECKING, Any, Callable, Dict, List, Optional, Tuple, cast
25
- from urllib.parse import parse_qs, urlencode, urlparse, urlunparse
24
+ from typing import TYPE_CHECKING, Any, Dict, List, Optional
25
+ from urllib.parse import (
26
+ parse_qs,
27
+ quote,
28
+ urlencode,
29
+ urlparse,
30
+ urlsplit,
31
+ urlunparse,
32
+ urlunsplit,
33
+ )
26
34
 
27
- import dateutil.parser
28
35
  import geojson
29
- import shapefile
30
- from dateutil import tz
31
- from dateutil.relativedelta import relativedelta
32
36
  from jsonpath_ng.jsonpath import Child
33
- from shapely.geometry import shape
34
- from shapely.geometry.base import BaseGeometry
35
- from shapely.ops import unary_union
36
37
 
37
38
  from eodag.api.product.metadata_mapping import (
38
39
  DEFAULT_METADATA_MAPPING,
@@ -42,7 +43,6 @@ from eodag.api.product.metadata_mapping import (
42
43
  from eodag.rest.config import Settings
43
44
  from eodag.rest.utils.rfc3339 import str_to_interval
44
45
  from eodag.utils import (
45
- DEFAULT_MISSION_START_DATE,
46
46
  deepcopy,
47
47
  dict_items_recursive_apply,
48
48
  format_dict_items,
@@ -50,14 +50,12 @@ from eodag.utils import (
50
50
  jsonpath_parse_dict_items,
51
51
  string_to_jsonpath,
52
52
  update_nested_dict,
53
- urljoin,
54
53
  )
55
54
  from eodag.utils.exceptions import (
56
55
  NoMatchingProductType,
57
56
  NotAvailableError,
58
57
  RequestError,
59
58
  TimeOutError,
60
- ValidationError,
61
59
  )
62
60
  from eodag.utils.requests import fetch_json
63
61
 
@@ -69,8 +67,6 @@ if TYPE_CHECKING:
69
67
 
70
68
  logger = logging.getLogger("eodag.rest.stac")
71
69
 
72
- STAC_CATALOGS_PREFIX = "catalogs"
73
-
74
70
  # fields not to put in item properties
75
71
  COLLECTION_PROPERTIES = [
76
72
  "abstract",
@@ -102,19 +98,21 @@ IGNORED_ITEM_PROPERTIES = [
102
98
  ]
103
99
 
104
100
 
101
+ def _quote_url_path(url: str) -> str:
102
+ parsed = urlsplit(url)
103
+ path = quote(parsed.path)
104
+ components = (parsed.scheme, parsed.netloc, path, parsed.query, parsed.fragment)
105
+ return urlunsplit(components)
106
+
107
+
105
108
  class StacCommon:
106
109
  """Stac common object
107
110
 
108
111
  :param url: Requested URL
109
- :type url: str
110
112
  :param stac_config: STAC configuration from stac.yml conf file
111
- :type stac_config: dict
112
113
  :param provider: (optional) Chosen provider
113
- :type provider: str
114
114
  :param eodag_api: EODAG python API instance
115
- :type eodag_api: :class:`eodag.api.core.EODataAccessGateway`
116
115
  :param root: (optional) API root
117
- :type root: str
118
116
  """
119
117
 
120
118
  def __init__(
@@ -137,7 +135,6 @@ class StacCommon:
137
135
  """Updates data using given input STAC dict data
138
136
 
139
137
  :param data: Catalog data (parsed STAC dict)
140
- :type data: dict
141
138
  """
142
139
  self.data.update(data)
143
140
 
@@ -149,16 +146,18 @@ class StacCommon:
149
146
  ):
150
147
  for i, bbox in enumerate(self.data["extent"]["spatial"]["bbox"]):
151
148
  self.data["extent"]["spatial"]["bbox"][i] = [float(x) for x in bbox]
152
- # "None" values to None
153
- apply_method: Callable[[str, str], Optional[str]] = lambda _, v: (
154
- None if v == "None" else v
155
- )
156
- self.data = dict_items_recursive_apply(self.data, apply_method)
157
- # ids and titles as str
158
- apply_method: Callable[[str, str], Optional[str]] = lambda k, v: (
159
- str(v) if k in ["title", "id"] else v
160
- )
161
- self.data = dict_items_recursive_apply(self.data, apply_method)
149
+
150
+ def apply_method_none(_: str, v: str) -> Optional[str]:
151
+ """ "None" values to None"""
152
+ return None if v == "None" else v
153
+
154
+ self.data = dict_items_recursive_apply(self.data, apply_method_none)
155
+
156
+ def apply_method_ids(k, v):
157
+ """ids and titles as str"""
158
+ return str(v) if k in ["title", "id"] else v
159
+
160
+ self.data = dict_items_recursive_apply(self.data, apply_method_ids)
162
161
 
163
162
  # empty stac_extensions: "" to []
164
163
  if not self.data.get("stac_extensions", True):
@@ -171,15 +170,10 @@ class StacCommon:
171
170
  """Parse STAC extension from config and return as dict
172
171
 
173
172
  :param url: Requested URL
174
- :type url: str
175
173
  :param stac_config: STAC configuration from stac.yml conf file
176
- :type stac_config: dict
177
174
  :param extension: Extension name
178
- :type extension: str
179
175
  :param kwargs: Additional variables needed for parsing extension
180
- :type kwargs: Any
181
- :returns: STAC extension as dictionnary
182
- :rtype: dict
176
+ :returns: STAC extension as dictionary
183
177
  """
184
178
  extension_model = deepcopy(stac_config).get("extensions", {}).get(extension, {})
185
179
 
@@ -211,15 +205,10 @@ class StacItem(StacCommon):
211
205
  """Stac item object
212
206
 
213
207
  :param url: Requested URL
214
- :type url: str
215
208
  :param stac_config: STAC configuration from stac.yml conf file
216
- :type stac_config: dict
217
209
  :param provider: (optional) Chosen provider
218
- :type provider: str
219
210
  :param eodag_api: EODAG python API instance
220
- :type eodag_api: :class:`eodag.api.core.EODataAccessGateway`
221
211
  :param root: (optional) API root
222
- :type root: str
223
212
  """
224
213
 
225
214
  def __init__(
@@ -244,11 +233,8 @@ class StacItem(StacCommon):
244
233
  """Build STAC items list from EODAG search results
245
234
 
246
235
  :param search_results: EODAG search results
247
- :type search_results: :class:`~eodag.api.search_result.SearchResult`
248
236
  :param catalog: STAC catalog dict used for parsing item metadata
249
- :type catalog: dict
250
237
  :returns: STAC item dicts list
251
- :rtype: list
252
238
  """
253
239
  if len(search_results) <= 0:
254
240
  return []
@@ -359,6 +345,10 @@ class StacItem(StacCommon):
359
345
  # remove empty properties
360
346
  product_item = self.__filter_item_properties_values(product_item)
361
347
 
348
+ # quote invalid characters in links
349
+ for link in product_item["links"]:
350
+ link["href"] = _quote_url_path(link["href"])
351
+
362
352
  # update item link with datacube query-string
363
353
  if _dc_qs or self.provider:
364
354
  url_parts = urlparse(str(product_item["links"][0]["href"]))
@@ -395,9 +385,12 @@ class StacItem(StacCommon):
395
385
  origin_href = product.remote_location
396
386
 
397
387
  # update download link with up-to-date query-args
388
+ quoted_href = _quote_url_path(
389
+ downloadlink_href
390
+ ) # quote invalid characters in url
398
391
  assets["downloadLink"] = {
399
392
  "title": "Download link",
400
- "href": downloadlink_href,
393
+ "href": quoted_href,
401
394
  "type": "application/zip",
402
395
  }
403
396
 
@@ -441,6 +434,7 @@ class StacItem(StacCommon):
441
434
  assets[asset_key]["type"] = asset_type
442
435
  if origin := assets[asset_key].get("alternate", {}).get("origin"):
443
436
  origin["type"] = asset_type
437
+ asset_value["href"] = _quote_url_path(asset_value["href"])
444
438
 
445
439
  if thumbnail_url := product.properties.get(
446
440
  "quicklook", product.properties.get("thumbnail", None)
@@ -464,11 +458,8 @@ class StacItem(StacCommon):
464
458
  """Build STAC items from EODAG search results
465
459
 
466
460
  :param search_results: EODAG search results
467
- :type search_results: :class:`~eodag.api.search_result.SearchResult`
468
461
  :param catalog: STAC catalog dict used for parsing item metadata
469
- :type catalog: dict
470
- :returns: Items dictionnary
471
- :rtype: dict
462
+ :returns: Items dictionary
472
463
  """
473
464
  items_model = deepcopy(self.stac_config["items"])
474
465
 
@@ -519,11 +510,8 @@ class StacItem(StacCommon):
519
510
  part of oseo extension.
520
511
 
521
512
  :param item_model: Item model from stac_config
522
- :type item_model: dict
523
513
  :param product_type: Product type
524
- :type product_type: str
525
514
  :returns: Filtered item model
526
- :rtype: dict
527
515
  """
528
516
  try:
529
517
  product_type_dict = [
@@ -584,9 +572,7 @@ class StacItem(StacCommon):
584
572
  """Removes empty properties, unused extensions, and add missing extensions
585
573
 
586
574
  :param item: STAC item data
587
- :type item: dict
588
575
  :returns: Filtered item model
589
- :rtype: dict
590
576
  """
591
577
  all_extensions_dict: Dict[str, str] = deepcopy(
592
578
  self.stac_config["stac_extensions"]
@@ -617,9 +603,7 @@ class StacItem(StacCommon):
617
603
  """Build STAC item from EODAG product
618
604
 
619
605
  :param product: EODAG product
620
- :type product: :class:`eodag.api.product._product.EOProduct`
621
606
  :returns: STAC item
622
- :rtype: list
623
607
  """
624
608
  product_type = str(product.product_type)
625
609
 
@@ -633,7 +617,7 @@ class StacItem(StacCommon):
633
617
  root=self.root,
634
618
  provider=self.provider,
635
619
  eodag_api=self.eodag_api,
636
- catalogs=[product_type],
620
+ collection=product_type,
637
621
  )
638
622
 
639
623
  product_dict = deepcopy(product.__dict__)
@@ -668,15 +652,10 @@ class StacCollection(StacCommon):
668
652
  """Stac collection object
669
653
 
670
654
  :param url: Requested URL
671
- :type url: str
672
655
  :param stac_config: STAC configuration from stac.yml conf file
673
- :type stac_config: dict
674
656
  :param provider: (optional) Chosen provider
675
- :type provider: str
676
657
  :param eodag_api: EODAG python API instance
677
- :type eodag_api: :class:`eodag.api.core.EODataAccessGateway`
678
658
  :param root: (optional) API root
679
- :type root: str
680
659
  """
681
660
 
682
661
  # External STAC collections
@@ -687,7 +666,6 @@ class StacCollection(StacCommon):
687
666
  """Load external STAC collections
688
667
 
689
668
  :param eodag_api: EODAG python API instance
690
- :type eodag_api: :class:`eodag.api.core.EODataAccessGateway`
691
669
  """
692
670
  list_product_types = eodag_api.list_product_types(fetch_providers=False)
693
671
  for product_type in list_product_types:
@@ -727,9 +705,7 @@ class StacCollection(StacCommon):
727
705
  """Retrieve a list of providers for a given product type.
728
706
 
729
707
  :param product_type: Dictionary containing information about the product type.
730
- :type product_type: dict
731
708
  :return: A list of provider names.
732
- :rtype: list
733
709
  """
734
710
  if self.provider:
735
711
  return [self.provider]
@@ -747,11 +723,8 @@ class StacCollection(StacCommon):
747
723
  """Generate a STAC collection dictionary for a given product type.
748
724
 
749
725
  :param collection_model: The base model for the STAC collection.
750
- :type collection_model: Any
751
726
  :param product_type: Dictionary containing information about the product type.
752
- :type product_type: dict
753
727
  :return: A dictionary representing the STAC collection for the given product type.
754
- :rtype: dict
755
728
  """
756
729
  providers = self.__list_product_type_providers(product_type)
757
730
 
@@ -782,25 +755,34 @@ class StacCollection(StacCommon):
782
755
  ]
783
756
  ext_stac_collection["links"].append(link)
784
757
 
758
+ # merge "summaries"
759
+ ext_stac_collection["summaries"] = {
760
+ k: v
761
+ for k, v in {
762
+ **ext_stac_collection.get("summaries", {}),
763
+ **product_type_collection["summaries"],
764
+ }.items()
765
+ if v and any(v)
766
+ }
767
+
785
768
  # merge "keywords" lists
786
- if "keywords" in ext_stac_collection:
787
- try:
788
- ext_stac_collection["keywords"] = [
789
- k
790
- for k in set(
791
- ext_stac_collection["keywords"]
792
- + product_type_collection["keywords"]
793
- )
794
- if k is not None
795
- ]
796
- except TypeError as e:
797
- logger.warning(
798
- f"Could not merge keywords from external collection for {product_type['ID']}: {str(e)}"
799
- )
800
- logger.debug(
801
- f"External collection keywords: {str(ext_stac_collection['keywords'])}, ",
802
- f"Product type keywords: {str(product_type_collection['keywords'])}",
769
+ try:
770
+ ext_stac_collection["keywords"] = [
771
+ k
772
+ for k in set(
773
+ ext_stac_collection.get("keywords", [])
774
+ + product_type_collection["keywords"]
803
775
  )
776
+ if k is not None
777
+ ]
778
+ except TypeError as e:
779
+ logger.warning(
780
+ f"Could not merge keywords from external collection for {product_type['ID']}: {str(e)}"
781
+ )
782
+ logger.debug(
783
+ f"External collection keywords: {str(ext_stac_collection.get('keywords'))}, ",
784
+ f"Product type keywords: {str(product_type_collection['keywords'])}",
785
+ )
804
786
 
805
787
  # merge providers
806
788
  if "providers" in ext_stac_collection:
@@ -837,9 +819,7 @@ class StacCollection(StacCommon):
837
819
  """Build STAC collections list
838
820
 
839
821
  :param filters: (optional) Additional filters for collections search
840
- :type filters: dict
841
822
  :returns: STAC collection dicts list
842
- :rtype: list
843
823
  """
844
824
  collection_model = deepcopy(self.stac_config["collection"])
845
825
 
@@ -884,17 +864,11 @@ class StacCatalog(StacCommon):
884
864
  """Stac Catalog object
885
865
 
886
866
  :param url: Requested URL
887
- :type url: str
888
867
  :param stac_config: STAC configuration from stac.yml conf file
889
- :type stac_config: dict
890
868
  :param provider: Chosen provider
891
- :type provider: (optional) str
892
869
  :param eodag_api: EODAG python API instance
893
- :type eodag_api: :class:`eodag.api.core.EODataAccessGateway`
894
870
  :param root: (optional) API root
895
- :type root: str
896
- :param catalogs: (optional) Catalogs list
897
- :type catalogs: list
871
+ :param collection: (optional) product type id
898
872
  """
899
873
 
900
874
  def __init__(
@@ -904,7 +878,7 @@ class StacCatalog(StacCommon):
904
878
  provider: Optional[str],
905
879
  eodag_api: EODataAccessGateway,
906
880
  root: str = "/",
907
- catalogs: Optional[List[str]] = None,
881
+ collection: Optional[str] = None,
908
882
  ) -> None:
909
883
  super(StacCatalog, self).__init__(
910
884
  url=url,
@@ -931,13 +905,12 @@ class StacCatalog(StacCommon):
931
905
  self.data["links"] += self.children
932
906
 
933
907
  # build catalog
934
- self.__build_stac_catalog(catalogs)
908
+ self.__build_stac_catalog(collection)
935
909
 
936
910
  def __update_data_from_catalog_config(self, catalog_config: Dict[str, Any]) -> bool:
937
911
  """Updates configuration and data using given input catalog config
938
912
 
939
913
  :param catalog_config: Catalog config, from yml stac_config[catalogs]
940
- :type catalog_config: dict
941
914
  """
942
915
  model = catalog_config["model"]
943
916
 
@@ -957,18 +930,32 @@ class StacCatalog(StacCommon):
957
930
 
958
931
  return True
959
932
 
960
- def set_children(self, children: Optional[List[Dict[str, Any]]] = None) -> bool:
961
- """Set catalog children / links
933
+ def __build_stac_catalog(self, collection: Optional[str] = None) -> StacCatalog:
934
+ """Build nested catalog from catalag list
962
935
 
963
- :param children: (optional) Children list
964
- :type children: list
936
+ :param collection: (optional) product type id
937
+ :returns: This catalog obj
965
938
  """
966
- self.children = children or []
967
- self.data["links"] = [
968
- link for link in self.data["links"] if link["rel"] != "child"
969
- ]
970
- self.data["links"] += self.children
971
- return True
939
+ settings = Settings.from_environment()
940
+
941
+ if not collection:
942
+ # Build root catalog combined with landing page
943
+ self.__update_data_from_catalog_config(
944
+ {
945
+ "model": {
946
+ **deepcopy(self.stac_config["landing_page"]),
947
+ **{
948
+ "provider": self.provider,
949
+ "id": settings.stac_api_landing_id,
950
+ "title": settings.stac_api_title,
951
+ "description": settings.stac_api_description,
952
+ },
953
+ }
954
+ }
955
+ )
956
+ else:
957
+ self.set_stac_product_type_by_id(collection)
958
+ return self
972
959
 
973
960
  def set_stac_product_type_by_id(
974
961
  self, product_type: str, **_: Any
@@ -976,7 +963,6 @@ class StacCatalog(StacCommon):
976
963
  """Updates catalog with given product_type
977
964
 
978
965
  :param product_type: Product type
979
- :type product_type: str
980
966
  """
981
967
  collections = StacCollection(
982
968
  url=self.url,
@@ -989,7 +975,17 @@ class StacCatalog(StacCommon):
989
975
  if not collections:
990
976
  raise NotAvailableError(f"Collection {product_type} does not exist.")
991
977
 
992
- cat_model = deepcopy(self.stac_config["catalogs"]["product_type"]["model"])
978
+ cat_model = {
979
+ "id": "{collection[id]}",
980
+ "title": "{collection[title]}",
981
+ "description": "{collection[description]}",
982
+ "extent": "{collection[extent]}",
983
+ "crs": "http://www.opengis.net/def/crs/OGC/1.3/CRS84",
984
+ "keywords": "{collection[keywords]}",
985
+ "license": "{collection[license]}",
986
+ "providers": "{collection[providers]}",
987
+ "summaries": "{collection[summaries]}",
988
+ }
993
989
  # parse f-strings
994
990
  format_args = deepcopy(self.stac_config)
995
991
  format_args["catalog"] = defaultdict(str, **self.data)
@@ -1006,524 +1002,3 @@ class StacCatalog(StacCommon):
1006
1002
  self.search_args.update({"productType": product_type})
1007
1003
 
1008
1004
  return parsed_dict
1009
-
1010
- # get / set dates filters -------------------------------------------------
1011
-
1012
- def get_stac_years_list(self, **_: Any) -> List[int]:
1013
- """Get catalog available years list
1014
-
1015
- :returns: Years list
1016
- :rtype: list
1017
- """
1018
- extent_date_min, extent_date_max = self.get_datetime_extent()
1019
-
1020
- return list(range(extent_date_min.year, extent_date_max.year + 1))
1021
-
1022
- def get_stac_months_list(self, **_: Any) -> List[int]:
1023
- """Get catalog available months list
1024
-
1025
- :returns: Months list
1026
- :rtype: list
1027
- """
1028
- extent_date_min, extent_date_max = self.get_datetime_extent()
1029
-
1030
- return list(
1031
- range(
1032
- extent_date_min.month,
1033
- (extent_date_max - relativedelta(days=1)).month + 1,
1034
- )
1035
- )
1036
-
1037
- def get_stac_days_list(self, **_: Any) -> List[int]:
1038
- """Get catalog available days list
1039
-
1040
- :returns: Days list
1041
- :rtype: list
1042
- """
1043
- extent_date_min, extent_date_max = self.get_datetime_extent()
1044
-
1045
- return list(
1046
- range(
1047
- extent_date_min.day, (extent_date_max - relativedelta(days=1)).day + 1
1048
- )
1049
- )
1050
-
1051
- def set_stac_year_by_id(self, year: str, **_: Any) -> Dict[str, Any]:
1052
- """Updates and returns catalog with given year
1053
-
1054
- :param year: Year number
1055
- :type year: str
1056
- :returns: Updated catalog
1057
- :rtype: dict
1058
- """
1059
- extent_date_min, extent_date_max = self.get_datetime_extent()
1060
-
1061
- datetime_min = max(
1062
- [extent_date_min, dateutil.parser.parse(f"{year}-01-01T00:00:00Z")]
1063
- )
1064
- datetime_max = min(
1065
- [
1066
- extent_date_max,
1067
- dateutil.parser.parse(f"{year}-01-01T00:00:00Z")
1068
- + relativedelta(years=1),
1069
- ]
1070
- )
1071
-
1072
- catalog_model = deepcopy(self.stac_config["catalogs"]["year"]["model"])
1073
-
1074
- parsed_dict = self.set_stac_date(datetime_min, datetime_max, catalog_model)
1075
-
1076
- return parsed_dict
1077
-
1078
- def set_stac_month_by_id(self, month: str, **_: Any) -> Dict[str, Any]:
1079
- """Updates and returns catalog with given month
1080
-
1081
- :param month: Month number
1082
- :type month: str
1083
- :returns: Updated catalog
1084
- :rtype: dict
1085
- """
1086
- extent_date_min, extent_date_max = self.get_datetime_extent()
1087
- year = extent_date_min.year
1088
-
1089
- datetime_min = max(
1090
- [
1091
- extent_date_min,
1092
- dateutil.parser.parse(f"{year}-{month}-01T00:00:00Z"),
1093
- ]
1094
- )
1095
- datetime_max = min(
1096
- [
1097
- extent_date_max,
1098
- dateutil.parser.parse(f"{year}-{month}-01T00:00:00Z")
1099
- + relativedelta(months=1),
1100
- ]
1101
- )
1102
-
1103
- catalog_model = deepcopy(self.stac_config["catalogs"]["month"]["model"])
1104
-
1105
- parsed_dict = self.set_stac_date(datetime_min, datetime_max, catalog_model)
1106
-
1107
- return parsed_dict
1108
-
1109
- def set_stac_day_by_id(self, day: str, **_: Any) -> Dict[str, Any]:
1110
- """Updates and returns catalog with given day
1111
-
1112
- :param day: Day number
1113
- :type day: str
1114
- :returns: Updated catalog
1115
- :rtype: dict
1116
- """
1117
- extent_date_min, extent_date_max = self.get_datetime_extent()
1118
- year = extent_date_min.year
1119
- month = extent_date_min.month
1120
-
1121
- datetime_min = max(
1122
- [
1123
- extent_date_min,
1124
- dateutil.parser.parse(f"{year}-{month}-{day}T00:00:00Z"),
1125
- ]
1126
- )
1127
- datetime_max = min(
1128
- [
1129
- extent_date_max,
1130
- dateutil.parser.parse(f"{year}-{month}-{day}T00:00:00Z")
1131
- + relativedelta(days=1),
1132
- ]
1133
- )
1134
-
1135
- catalog_model = deepcopy(self.stac_config["catalogs"]["day"]["model"])
1136
-
1137
- parsed_dict = self.set_stac_date(datetime_min, datetime_max, catalog_model)
1138
-
1139
- return parsed_dict
1140
-
1141
- def get_datetime_extent(self) -> Tuple[datetime, datetime]:
1142
- """Returns catalog temporal extent as datetime objs
1143
-
1144
- :returns: Start & stop dates
1145
- :rtype: tuple
1146
- """
1147
- extent_date_min = dateutil.parser.parse(DEFAULT_MISSION_START_DATE).replace(
1148
- tzinfo=tz.UTC
1149
- )
1150
- extent_date_max = datetime.now(timezone.utc).replace(tzinfo=tz.UTC)
1151
- for interval in self.data["extent"]["temporal"]["interval"]:
1152
- extent_date_min_str, extent_date_max_str = interval
1153
- # date min
1154
- if extent_date_min_str:
1155
- extent_date_min = max(
1156
- extent_date_min, dateutil.parser.parse(extent_date_min_str)
1157
- )
1158
- # date max
1159
- if extent_date_max_str:
1160
- extent_date_max = min(
1161
- extent_date_max, dateutil.parser.parse(extent_date_max_str)
1162
- )
1163
-
1164
- return (
1165
- extent_date_min.replace(tzinfo=tz.UTC),
1166
- extent_date_max.replace(tzinfo=tz.UTC),
1167
- )
1168
-
1169
- def set_stac_date(
1170
- self,
1171
- datetime_min: datetime,
1172
- datetime_max: datetime,
1173
- catalog_model: Dict[str, Any],
1174
- ):
1175
- """Updates catalog data using given dates
1176
-
1177
- :param datetime_min: Date min of interval
1178
- :type datetime_min: :class:`datetime`
1179
- :param datetime_max: Date max of interval
1180
- :type datetime_max: :class:`datetime`
1181
- :param catalog_model: Catalog model to use, from yml stac_config[catalogs]
1182
- :type catalog_model: dict
1183
- :returns: Updated catalog
1184
- :rtype: dict
1185
- """
1186
- # parse f-strings
1187
- format_args = deepcopy(self.stac_config)
1188
- format_args["catalog"] = defaultdict(str, **self.data)
1189
- format_args["date"] = defaultdict(
1190
- str,
1191
- {
1192
- "year": datetime_min.year,
1193
- "month": datetime_min.month,
1194
- "day": datetime_min.day,
1195
- "min": datetime_min.isoformat().replace("+00:00", "Z"),
1196
- "max": datetime_max.isoformat().replace("+00:00", "Z"),
1197
- },
1198
- )
1199
- parsed_dict: Dict[str, Any] = format_dict_items(catalog_model, **format_args)
1200
-
1201
- self.update_data(parsed_dict)
1202
-
1203
- # update search args
1204
- self.search_args.update(
1205
- {
1206
- "start": datetime_min.isoformat().replace("+00:00", "Z"),
1207
- "end": datetime_max.isoformat().replace("+00:00", "Z"),
1208
- }
1209
- )
1210
- return parsed_dict
1211
-
1212
- # get / set cloud_cover filter --------------------------------------------
1213
-
1214
- def get_stac_cloud_covers_list(self, **_: Any) -> List[int]:
1215
- """Get cloud_cover list
1216
-
1217
- :returns: cloud_cover list
1218
- :rtype: list
1219
- """
1220
- return list(range(0, 101, 10))
1221
-
1222
- def set_stac_cloud_cover_by_id(self, cloud_cover: str, **_: Any) -> Dict[str, Any]:
1223
- """Updates and returns catalog with given max cloud_cover
1224
-
1225
- :param cloud_cover: Cloud_cover number
1226
- :type cloud_cover: str
1227
- :returns: Updated catalog
1228
- :rtype: dict
1229
- """
1230
- cat_model = deepcopy(self.stac_config["catalogs"]["cloud_cover"]["model"])
1231
- # parse f-strings
1232
- format_args = deepcopy(self.stac_config)
1233
- format_args["catalog"] = defaultdict(str, **self.data)
1234
- format_args["cloud_cover"] = cloud_cover
1235
- parsed_dict: Dict[str, Any] = format_dict_items(cat_model, **format_args)
1236
-
1237
- self.update_data(parsed_dict)
1238
-
1239
- # update search args
1240
- self.search_args.update({"cloudCover": cloud_cover})
1241
-
1242
- return parsed_dict
1243
-
1244
- # get / set locations filter ----------------------------------------------
1245
-
1246
- def get_stac_location_list(self, catalog_name: str) -> List[str]:
1247
- """Get locations list using stac_conf & locations_config
1248
-
1249
- :param catalog_name: Catalog/location name
1250
- :type catalog_name: str
1251
- :returns: Locations list
1252
- :rtype: list
1253
- """
1254
-
1255
- if catalog_name not in self.stac_config["catalogs"]:
1256
- logger.warning("no entry found for %s in location_config", catalog_name)
1257
- return []
1258
- location_config = self.stac_config["catalogs"][catalog_name]
1259
-
1260
- for k in ["path", "attr"]:
1261
- if k not in location_config.keys():
1262
- logger.warning(
1263
- "no %s key found for %s in location_config", k, catalog_name
1264
- )
1265
- return []
1266
- path = location_config["path"]
1267
- attr = location_config["attr"]
1268
-
1269
- with shapefile.Reader(path) as shp:
1270
- countries_list: List[str] = [rec[attr] for rec in shp.records()] # type: ignore
1271
-
1272
- # remove duplicates
1273
- countries_list = list(set(countries_list))
1274
-
1275
- countries_list.sort()
1276
-
1277
- return countries_list
1278
-
1279
- def set_stac_location_by_id(
1280
- self, location: str, catalog_name: str
1281
- ) -> Dict[str, Any]:
1282
- """Updates and returns catalog with given location
1283
-
1284
- :param location: Feature attribute value for shp filtering
1285
- :type location: str
1286
- :param catalog_name: Catalog/location name
1287
- :type catalog_name: str
1288
- :returns: Updated catalog
1289
- :rtype: dict
1290
- """
1291
- location_list_cat_key = catalog_name + "_list"
1292
-
1293
- if location_list_cat_key not in self.stac_config["catalogs"]:
1294
- logger.warning(
1295
- "no entry found for %s's list in location_config", catalog_name
1296
- )
1297
- return {}
1298
- location_config = self.stac_config["catalogs"][location_list_cat_key]
1299
-
1300
- for k in ["path", "attr"]:
1301
- if k not in location_config.keys():
1302
- logger.warning(
1303
- "no %s key found for %s's list in location_config", k, catalog_name
1304
- )
1305
- return {}
1306
- path = location_config["path"]
1307
- attr = location_config["attr"]
1308
-
1309
- with shapefile.Reader(path) as shp:
1310
- geom_hits = [
1311
- shape(shaperec.shape)
1312
- for shaperec in shp.shapeRecords()
1313
- if shaperec.record.as_dict().get(attr, None) == location
1314
- ]
1315
-
1316
- if not geom_hits:
1317
- logger.warning(
1318
- "no feature found in %s matching %s=%s", path, attr, location
1319
- )
1320
- return {}
1321
-
1322
- geom = cast(BaseGeometry, unary_union(geom_hits))
1323
-
1324
- cat_model = deepcopy(self.stac_config["catalogs"]["country"]["model"])
1325
- # parse f-strings
1326
- format_args = deepcopy(self.stac_config)
1327
- format_args["catalog"] = defaultdict(str, **self.data)
1328
- format_args["feature"] = defaultdict(str, {"geometry": geom, "id": location})
1329
- parsed_dict: Dict[str, Any] = format_dict_items(cat_model, **format_args)
1330
-
1331
- self.update_data(parsed_dict)
1332
-
1333
- # update search args
1334
- self.search_args.update({"geom": geom})
1335
-
1336
- return parsed_dict
1337
-
1338
- def build_locations_config(self) -> Dict[str, str]:
1339
- """Build locations config from stac_conf[locations_catalogs] & eodag_api.locations_config
1340
-
1341
- :returns: Locations configuration dict
1342
- :rtype: dict
1343
- """
1344
- user_config_locations_list = self.eodag_api.locations_config
1345
-
1346
- locations_config_model = deepcopy(self.stac_config["locations_catalogs"])
1347
-
1348
- locations_config: Dict[str, str] = {}
1349
- for loc in user_config_locations_list:
1350
- # parse jsonpath
1351
- parsed = jsonpath_parse_dict_items(
1352
- locations_config_model, {"shp_location": loc}
1353
- )
1354
-
1355
- # set default child/parent for this location
1356
- parsed["location"]["parent_key"] = f"{loc['name']}_list"
1357
-
1358
- locations_config[f"{loc['name']}_list"] = parsed["locations_list"]
1359
- locations_config[loc["name"]] = parsed["location"]
1360
-
1361
- return locations_config
1362
-
1363
- def __build_stac_catalog(self, catalogs: Optional[List[str]] = None) -> StacCatalog:
1364
- """Build nested catalog from catalag list
1365
-
1366
- :param catalogs: (optional) Catalogs list
1367
- :type catalogs: list
1368
- :returns: This catalog obj
1369
- :rtype: :class:`eodag.stac.StacCatalog`
1370
- """
1371
- settings = Settings.from_environment()
1372
-
1373
- # update conf with user shp locations
1374
- locations_config = self.build_locations_config()
1375
-
1376
- self.stac_config["catalogs"] = {
1377
- **deepcopy(self.stac_config["catalogs"]),
1378
- **locations_config,
1379
- }
1380
-
1381
- if not catalogs:
1382
- # Build root catalog combined with landing page
1383
- self.__update_data_from_catalog_config(
1384
- {
1385
- "model": {
1386
- **deepcopy(self.stac_config["landing_page"]),
1387
- **{
1388
- "provider": self.provider,
1389
- "id": settings.stac_api_landing_id,
1390
- "title": settings.stac_api_title,
1391
- "description": settings.stac_api_description,
1392
- },
1393
- }
1394
- }
1395
- )
1396
-
1397
- # build children : product_types
1398
- product_types_list = [
1399
- pt
1400
- for pt in self.eodag_api.list_product_types(
1401
- provider=self.provider, fetch_providers=False
1402
- )
1403
- ]
1404
- self.set_children(
1405
- [
1406
- {
1407
- "rel": "child",
1408
- "href": urljoin(
1409
- self.url, f"{STAC_CATALOGS_PREFIX}/{product_type['ID']}"
1410
- ),
1411
- "title": product_type["title"],
1412
- }
1413
- for product_type in product_types_list
1414
- ]
1415
- )
1416
- return self
1417
-
1418
- # use product_types_list as base for building nested catalogs
1419
- self.__update_data_from_catalog_config(
1420
- deepcopy(self.stac_config["catalogs"]["product_types_list"])
1421
- )
1422
-
1423
- for idx, cat in enumerate(catalogs):
1424
- if idx % 2 == 0:
1425
- # even: cat is a filtering value ----------------------------------
1426
- cat_data_name = self.catalog_config["child_key"]
1427
- cat_data_value = cat
1428
-
1429
- # update data
1430
- cat_data_name_dict = self.stac_config["catalogs"][cat_data_name]
1431
- set_data_method_name = (
1432
- f"set_stac_{cat_data_name}_by_id"
1433
- if "catalog_type" not in cat_data_name_dict.keys()
1434
- else f"set_stac_{cat_data_name_dict['catalog_type']}_by_id"
1435
- )
1436
- set_data_method = getattr(self, set_data_method_name)
1437
- set_data_method(cat_data_value, catalog_name=cat_data_name)
1438
-
1439
- if idx == len(catalogs) - 1:
1440
- # build children : remaining filtering keys
1441
- remaining_catalogs_list = [
1442
- c
1443
- for c in self.stac_config["catalogs"].keys()
1444
- # keep filters not used yet AND
1445
- if self.stac_config["catalogs"][c]["model"]["id"]
1446
- not in catalogs
1447
- and (
1448
- # filters with no parent_key constraint (no key, or key=None) OR
1449
- "parent_key" not in self.stac_config["catalogs"][c]
1450
- or not self.stac_config["catalogs"][c]["parent_key"]
1451
- # filters matching parent_key constraint
1452
- or self.stac_config["catalogs"][c]["parent_key"]
1453
- == cat_data_name
1454
- )
1455
- # AND filters that match parent attr constraint (locations)
1456
- and (
1457
- "parent" not in self.stac_config["catalogs"][c]
1458
- or not self.stac_config["catalogs"][c]["parent"]["key"]
1459
- or (
1460
- self.stac_config["catalogs"][c]["parent"]["key"]
1461
- == cat_data_name
1462
- and self.stac_config["catalogs"][c]["parent"]["attr"]
1463
- == cat_data_value
1464
- )
1465
- )
1466
- ]
1467
-
1468
- self.set_children(
1469
- [
1470
- {
1471
- "rel": "child",
1472
- "href": self.url
1473
- + "/"
1474
- + self.stac_config["catalogs"][c]["model"]["id"],
1475
- "title": str(
1476
- self.stac_config["catalogs"][c]["model"]["id"]
1477
- ),
1478
- }
1479
- for c in remaining_catalogs_list
1480
- ]
1481
- + [
1482
- {
1483
- "rel": "items",
1484
- "href": self.url + "/items",
1485
- "title": "items",
1486
- }
1487
- ]
1488
- )
1489
-
1490
- else:
1491
- # odd: cat is a filtering key -------------------------------------
1492
- try:
1493
- cat_key = [
1494
- c
1495
- for c in self.stac_config["catalogs"].keys()
1496
- if self.stac_config["catalogs"][c]["model"]["id"] == cat
1497
- ][0]
1498
- except IndexError as e:
1499
- raise ValidationError(
1500
- f"Bad settings for {cat} in stac_config catalogs"
1501
- ) from e
1502
- cat_config = deepcopy(self.stac_config["catalogs"][cat_key])
1503
- # update data
1504
- self.__update_data_from_catalog_config(cat_config)
1505
-
1506
- # get filtering values list
1507
- get_data_method_name = (
1508
- f"get_stac_{cat_key}"
1509
- if "catalog_type"
1510
- not in self.stac_config["catalogs"][cat_key].keys()
1511
- else f"get_stac_{self.stac_config['catalogs'][cat_key]['catalog_type']}"
1512
- )
1513
- get_data_method = getattr(self, get_data_method_name)
1514
- cat_data_list = get_data_method(catalog_name=cat_key)
1515
-
1516
- if idx == len(catalogs) - 1:
1517
- # filtering values list as children (do not include items)
1518
- self.set_children(
1519
- [
1520
- {
1521
- "rel": "child",
1522
- "href": self.url + "/" + str(filtering_data),
1523
- "title": str(filtering_data),
1524
- }
1525
- for filtering_data in cat_data_list
1526
- ]
1527
- )
1528
-
1529
- return self