eodag 3.0.0b3__py3-none-any.whl → 3.0.1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- eodag/api/core.py +189 -125
- eodag/api/product/metadata_mapping.py +12 -3
- eodag/api/search_result.py +29 -3
- eodag/cli.py +35 -19
- eodag/config.py +412 -116
- eodag/plugins/apis/base.py +10 -4
- eodag/plugins/apis/ecmwf.py +14 -4
- eodag/plugins/apis/usgs.py +25 -2
- eodag/plugins/authentication/aws_auth.py +14 -5
- eodag/plugins/authentication/base.py +10 -1
- eodag/plugins/authentication/generic.py +14 -3
- eodag/plugins/authentication/header.py +12 -4
- eodag/plugins/authentication/keycloak.py +41 -22
- eodag/plugins/authentication/oauth.py +11 -1
- eodag/plugins/authentication/openid_connect.py +178 -163
- eodag/plugins/authentication/qsauth.py +12 -4
- eodag/plugins/authentication/sas_auth.py +19 -2
- eodag/plugins/authentication/token.py +57 -10
- eodag/plugins/authentication/token_exchange.py +19 -19
- eodag/plugins/crunch/base.py +4 -1
- eodag/plugins/crunch/filter_date.py +5 -2
- eodag/plugins/crunch/filter_latest_intersect.py +5 -4
- eodag/plugins/crunch/filter_latest_tpl_name.py +1 -1
- eodag/plugins/crunch/filter_overlap.py +5 -7
- eodag/plugins/crunch/filter_property.py +4 -3
- eodag/plugins/download/aws.py +39 -22
- eodag/plugins/download/base.py +11 -11
- eodag/plugins/download/creodias_s3.py +11 -2
- eodag/plugins/download/http.py +86 -52
- eodag/plugins/download/s3rest.py +20 -18
- eodag/plugins/manager.py +168 -23
- eodag/plugins/search/base.py +33 -14
- eodag/plugins/search/build_search_result.py +55 -51
- eodag/plugins/search/cop_marine.py +112 -29
- eodag/plugins/search/creodias_s3.py +20 -5
- eodag/plugins/search/csw.py +41 -1
- eodag/plugins/search/data_request_search.py +109 -9
- eodag/plugins/search/qssearch.py +532 -152
- eodag/plugins/search/static_stac_search.py +20 -21
- eodag/resources/ext_product_types.json +1 -1
- eodag/resources/product_types.yml +187 -56
- eodag/resources/providers.yml +1610 -1701
- eodag/resources/stac.yml +3 -163
- eodag/resources/user_conf_template.yml +112 -97
- eodag/rest/config.py +1 -2
- eodag/rest/constants.py +0 -1
- eodag/rest/core.py +61 -51
- eodag/rest/errors.py +181 -0
- eodag/rest/server.py +24 -325
- eodag/rest/stac.py +93 -544
- eodag/rest/types/eodag_search.py +13 -8
- eodag/rest/types/queryables.py +1 -2
- eodag/rest/types/stac_search.py +11 -2
- eodag/types/__init__.py +15 -3
- eodag/types/download_args.py +1 -1
- eodag/types/queryables.py +1 -2
- eodag/types/search_args.py +3 -3
- eodag/utils/__init__.py +77 -57
- eodag/utils/exceptions.py +23 -9
- eodag/utils/logging.py +37 -77
- eodag/utils/requests.py +1 -3
- eodag/utils/stac_reader.py +1 -1
- {eodag-3.0.0b3.dist-info → eodag-3.0.1.dist-info}/METADATA +11 -12
- eodag-3.0.1.dist-info/RECORD +109 -0
- {eodag-3.0.0b3.dist-info → eodag-3.0.1.dist-info}/WHEEL +1 -1
- {eodag-3.0.0b3.dist-info → eodag-3.0.1.dist-info}/entry_points.txt +1 -0
- eodag/resources/constraints/climate-dt.json +0 -13
- eodag/resources/constraints/extremes-dt.json +0 -8
- eodag-3.0.0b3.dist-info/RECORD +0 -110
- {eodag-3.0.0b3.dist-info → eodag-3.0.1.dist-info}/LICENSE +0 -0
- {eodag-3.0.0b3.dist-info → eodag-3.0.1.dist-info}/top_level.txt +0 -0
eodag/rest/stac.py
CHANGED
|
@@ -21,18 +21,19 @@ import logging
|
|
|
21
21
|
import os
|
|
22
22
|
from collections import defaultdict
|
|
23
23
|
from datetime import datetime, timezone
|
|
24
|
-
from typing import TYPE_CHECKING, Any, Dict, List, Optional
|
|
25
|
-
from urllib.parse import
|
|
24
|
+
from typing import TYPE_CHECKING, Any, Dict, List, Optional
|
|
25
|
+
from urllib.parse import (
|
|
26
|
+
parse_qs,
|
|
27
|
+
quote,
|
|
28
|
+
urlencode,
|
|
29
|
+
urlparse,
|
|
30
|
+
urlsplit,
|
|
31
|
+
urlunparse,
|
|
32
|
+
urlunsplit,
|
|
33
|
+
)
|
|
26
34
|
|
|
27
|
-
import dateutil.parser
|
|
28
35
|
import geojson
|
|
29
|
-
import shapefile
|
|
30
|
-
from dateutil import tz
|
|
31
|
-
from dateutil.relativedelta import relativedelta
|
|
32
36
|
from jsonpath_ng.jsonpath import Child
|
|
33
|
-
from shapely.geometry import shape
|
|
34
|
-
from shapely.geometry.base import BaseGeometry
|
|
35
|
-
from shapely.ops import unary_union
|
|
36
37
|
|
|
37
38
|
from eodag.api.product.metadata_mapping import (
|
|
38
39
|
DEFAULT_METADATA_MAPPING,
|
|
@@ -42,7 +43,6 @@ from eodag.api.product.metadata_mapping import (
|
|
|
42
43
|
from eodag.rest.config import Settings
|
|
43
44
|
from eodag.rest.utils.rfc3339 import str_to_interval
|
|
44
45
|
from eodag.utils import (
|
|
45
|
-
DEFAULT_MISSION_START_DATE,
|
|
46
46
|
deepcopy,
|
|
47
47
|
dict_items_recursive_apply,
|
|
48
48
|
format_dict_items,
|
|
@@ -50,14 +50,12 @@ from eodag.utils import (
|
|
|
50
50
|
jsonpath_parse_dict_items,
|
|
51
51
|
string_to_jsonpath,
|
|
52
52
|
update_nested_dict,
|
|
53
|
-
urljoin,
|
|
54
53
|
)
|
|
55
54
|
from eodag.utils.exceptions import (
|
|
56
55
|
NoMatchingProductType,
|
|
57
56
|
NotAvailableError,
|
|
58
57
|
RequestError,
|
|
59
58
|
TimeOutError,
|
|
60
|
-
ValidationError,
|
|
61
59
|
)
|
|
62
60
|
from eodag.utils.requests import fetch_json
|
|
63
61
|
|
|
@@ -69,8 +67,6 @@ if TYPE_CHECKING:
|
|
|
69
67
|
|
|
70
68
|
logger = logging.getLogger("eodag.rest.stac")
|
|
71
69
|
|
|
72
|
-
STAC_CATALOGS_PREFIX = "catalogs"
|
|
73
|
-
|
|
74
70
|
# fields not to put in item properties
|
|
75
71
|
COLLECTION_PROPERTIES = [
|
|
76
72
|
"abstract",
|
|
@@ -102,6 +98,13 @@ IGNORED_ITEM_PROPERTIES = [
|
|
|
102
98
|
]
|
|
103
99
|
|
|
104
100
|
|
|
101
|
+
def _quote_url_path(url: str) -> str:
|
|
102
|
+
parsed = urlsplit(url)
|
|
103
|
+
path = quote(parsed.path)
|
|
104
|
+
components = (parsed.scheme, parsed.netloc, path, parsed.query, parsed.fragment)
|
|
105
|
+
return urlunsplit(components)
|
|
106
|
+
|
|
107
|
+
|
|
105
108
|
class StacCommon:
|
|
106
109
|
"""Stac common object
|
|
107
110
|
|
|
@@ -170,7 +173,7 @@ class StacCommon:
|
|
|
170
173
|
:param stac_config: STAC configuration from stac.yml conf file
|
|
171
174
|
:param extension: Extension name
|
|
172
175
|
:param kwargs: Additional variables needed for parsing extension
|
|
173
|
-
:returns: STAC extension as
|
|
176
|
+
:returns: STAC extension as dictionary
|
|
174
177
|
"""
|
|
175
178
|
extension_model = deepcopy(stac_config).get("extensions", {}).get(extension, {})
|
|
176
179
|
|
|
@@ -342,6 +345,10 @@ class StacItem(StacCommon):
|
|
|
342
345
|
# remove empty properties
|
|
343
346
|
product_item = self.__filter_item_properties_values(product_item)
|
|
344
347
|
|
|
348
|
+
# quote invalid characters in links
|
|
349
|
+
for link in product_item["links"]:
|
|
350
|
+
link["href"] = _quote_url_path(link["href"])
|
|
351
|
+
|
|
345
352
|
# update item link with datacube query-string
|
|
346
353
|
if _dc_qs or self.provider:
|
|
347
354
|
url_parts = urlparse(str(product_item["links"][0]["href"]))
|
|
@@ -378,9 +385,12 @@ class StacItem(StacCommon):
|
|
|
378
385
|
origin_href = product.remote_location
|
|
379
386
|
|
|
380
387
|
# update download link with up-to-date query-args
|
|
388
|
+
quoted_href = _quote_url_path(
|
|
389
|
+
downloadlink_href
|
|
390
|
+
) # quote invalid characters in url
|
|
381
391
|
assets["downloadLink"] = {
|
|
382
392
|
"title": "Download link",
|
|
383
|
-
"href":
|
|
393
|
+
"href": quoted_href,
|
|
384
394
|
"type": "application/zip",
|
|
385
395
|
}
|
|
386
396
|
|
|
@@ -424,6 +434,7 @@ class StacItem(StacCommon):
|
|
|
424
434
|
assets[asset_key]["type"] = asset_type
|
|
425
435
|
if origin := assets[asset_key].get("alternate", {}).get("origin"):
|
|
426
436
|
origin["type"] = asset_type
|
|
437
|
+
asset_value["href"] = _quote_url_path(asset_value["href"])
|
|
427
438
|
|
|
428
439
|
if thumbnail_url := product.properties.get(
|
|
429
440
|
"quicklook", product.properties.get("thumbnail", None)
|
|
@@ -448,7 +459,7 @@ class StacItem(StacCommon):
|
|
|
448
459
|
|
|
449
460
|
:param search_results: EODAG search results
|
|
450
461
|
:param catalog: STAC catalog dict used for parsing item metadata
|
|
451
|
-
:returns: Items
|
|
462
|
+
:returns: Items dictionary
|
|
452
463
|
"""
|
|
453
464
|
items_model = deepcopy(self.stac_config["items"])
|
|
454
465
|
|
|
@@ -606,7 +617,7 @@ class StacItem(StacCommon):
|
|
|
606
617
|
root=self.root,
|
|
607
618
|
provider=self.provider,
|
|
608
619
|
eodag_api=self.eodag_api,
|
|
609
|
-
|
|
620
|
+
collection=product_type,
|
|
610
621
|
)
|
|
611
622
|
|
|
612
623
|
product_dict = deepcopy(product.__dict__)
|
|
@@ -744,25 +755,34 @@ class StacCollection(StacCommon):
|
|
|
744
755
|
]
|
|
745
756
|
ext_stac_collection["links"].append(link)
|
|
746
757
|
|
|
758
|
+
# merge "summaries"
|
|
759
|
+
ext_stac_collection["summaries"] = {
|
|
760
|
+
k: v
|
|
761
|
+
for k, v in {
|
|
762
|
+
**ext_stac_collection.get("summaries", {}),
|
|
763
|
+
**product_type_collection["summaries"],
|
|
764
|
+
}.items()
|
|
765
|
+
if v and any(v)
|
|
766
|
+
}
|
|
767
|
+
|
|
747
768
|
# merge "keywords" lists
|
|
748
|
-
|
|
749
|
-
|
|
750
|
-
|
|
751
|
-
|
|
752
|
-
|
|
753
|
-
|
|
754
|
-
+ product_type_collection["keywords"]
|
|
755
|
-
)
|
|
756
|
-
if k is not None
|
|
757
|
-
]
|
|
758
|
-
except TypeError as e:
|
|
759
|
-
logger.warning(
|
|
760
|
-
f"Could not merge keywords from external collection for {product_type['ID']}: {str(e)}"
|
|
761
|
-
)
|
|
762
|
-
logger.debug(
|
|
763
|
-
f"External collection keywords: {str(ext_stac_collection['keywords'])}, ",
|
|
764
|
-
f"Product type keywords: {str(product_type_collection['keywords'])}",
|
|
769
|
+
try:
|
|
770
|
+
ext_stac_collection["keywords"] = [
|
|
771
|
+
k
|
|
772
|
+
for k in set(
|
|
773
|
+
ext_stac_collection.get("keywords", [])
|
|
774
|
+
+ product_type_collection["keywords"]
|
|
765
775
|
)
|
|
776
|
+
if k is not None
|
|
777
|
+
]
|
|
778
|
+
except TypeError as e:
|
|
779
|
+
logger.warning(
|
|
780
|
+
f"Could not merge keywords from external collection for {product_type['ID']}: {str(e)}"
|
|
781
|
+
)
|
|
782
|
+
logger.debug(
|
|
783
|
+
f"External collection keywords: {str(ext_stac_collection.get('keywords'))}, ",
|
|
784
|
+
f"Product type keywords: {str(product_type_collection['keywords'])}",
|
|
785
|
+
)
|
|
766
786
|
|
|
767
787
|
# merge providers
|
|
768
788
|
if "providers" in ext_stac_collection:
|
|
@@ -848,7 +868,7 @@ class StacCatalog(StacCommon):
|
|
|
848
868
|
:param provider: Chosen provider
|
|
849
869
|
:param eodag_api: EODAG python API instance
|
|
850
870
|
:param root: (optional) API root
|
|
851
|
-
:param
|
|
871
|
+
:param collection: (optional) product type id
|
|
852
872
|
"""
|
|
853
873
|
|
|
854
874
|
def __init__(
|
|
@@ -858,7 +878,7 @@ class StacCatalog(StacCommon):
|
|
|
858
878
|
provider: Optional[str],
|
|
859
879
|
eodag_api: EODataAccessGateway,
|
|
860
880
|
root: str = "/",
|
|
861
|
-
|
|
881
|
+
collection: Optional[str] = None,
|
|
862
882
|
) -> None:
|
|
863
883
|
super(StacCatalog, self).__init__(
|
|
864
884
|
url=url,
|
|
@@ -885,7 +905,7 @@ class StacCatalog(StacCommon):
|
|
|
885
905
|
self.data["links"] += self.children
|
|
886
906
|
|
|
887
907
|
# build catalog
|
|
888
|
-
self.__build_stac_catalog(
|
|
908
|
+
self.__build_stac_catalog(collection)
|
|
889
909
|
|
|
890
910
|
def __update_data_from_catalog_config(self, catalog_config: Dict[str, Any]) -> bool:
|
|
891
911
|
"""Updates configuration and data using given input catalog config
|
|
@@ -910,17 +930,32 @@ class StacCatalog(StacCommon):
|
|
|
910
930
|
|
|
911
931
|
return True
|
|
912
932
|
|
|
913
|
-
def
|
|
914
|
-
"""
|
|
933
|
+
def __build_stac_catalog(self, collection: Optional[str] = None) -> StacCatalog:
|
|
934
|
+
"""Build nested catalog from catalag list
|
|
915
935
|
|
|
916
|
-
:param
|
|
936
|
+
:param collection: (optional) product type id
|
|
937
|
+
:returns: This catalog obj
|
|
917
938
|
"""
|
|
918
|
-
|
|
919
|
-
|
|
920
|
-
|
|
921
|
-
|
|
922
|
-
|
|
923
|
-
|
|
939
|
+
settings = Settings.from_environment()
|
|
940
|
+
|
|
941
|
+
if not collection:
|
|
942
|
+
# Build root catalog combined with landing page
|
|
943
|
+
self.__update_data_from_catalog_config(
|
|
944
|
+
{
|
|
945
|
+
"model": {
|
|
946
|
+
**deepcopy(self.stac_config["landing_page"]),
|
|
947
|
+
**{
|
|
948
|
+
"provider": self.provider,
|
|
949
|
+
"id": settings.stac_api_landing_id,
|
|
950
|
+
"title": settings.stac_api_title,
|
|
951
|
+
"description": settings.stac_api_description,
|
|
952
|
+
},
|
|
953
|
+
}
|
|
954
|
+
}
|
|
955
|
+
)
|
|
956
|
+
else:
|
|
957
|
+
self.set_stac_product_type_by_id(collection)
|
|
958
|
+
return self
|
|
924
959
|
|
|
925
960
|
def set_stac_product_type_by_id(
|
|
926
961
|
self, product_type: str, **_: Any
|
|
@@ -940,7 +975,17 @@ class StacCatalog(StacCommon):
|
|
|
940
975
|
if not collections:
|
|
941
976
|
raise NotAvailableError(f"Collection {product_type} does not exist.")
|
|
942
977
|
|
|
943
|
-
cat_model =
|
|
978
|
+
cat_model = {
|
|
979
|
+
"id": "{collection[id]}",
|
|
980
|
+
"title": "{collection[title]}",
|
|
981
|
+
"description": "{collection[description]}",
|
|
982
|
+
"extent": "{collection[extent]}",
|
|
983
|
+
"crs": "http://www.opengis.net/def/crs/OGC/1.3/CRS84",
|
|
984
|
+
"keywords": "{collection[keywords]}",
|
|
985
|
+
"license": "{collection[license]}",
|
|
986
|
+
"providers": "{collection[providers]}",
|
|
987
|
+
"summaries": "{collection[summaries]}",
|
|
988
|
+
}
|
|
944
989
|
# parse f-strings
|
|
945
990
|
format_args = deepcopy(self.stac_config)
|
|
946
991
|
format_args["catalog"] = defaultdict(str, **self.data)
|
|
@@ -957,499 +1002,3 @@ class StacCatalog(StacCommon):
|
|
|
957
1002
|
self.search_args.update({"productType": product_type})
|
|
958
1003
|
|
|
959
1004
|
return parsed_dict
|
|
960
|
-
|
|
961
|
-
# get / set dates filters -------------------------------------------------
|
|
962
|
-
|
|
963
|
-
def get_stac_years_list(self, **_: Any) -> List[int]:
|
|
964
|
-
"""Get catalog available years list
|
|
965
|
-
|
|
966
|
-
:returns: Years list
|
|
967
|
-
"""
|
|
968
|
-
extent_date_min, extent_date_max = self.get_datetime_extent()
|
|
969
|
-
|
|
970
|
-
return list(range(extent_date_min.year, extent_date_max.year + 1))
|
|
971
|
-
|
|
972
|
-
def get_stac_months_list(self, **_: Any) -> List[int]:
|
|
973
|
-
"""Get catalog available months list
|
|
974
|
-
|
|
975
|
-
:returns: Months list
|
|
976
|
-
"""
|
|
977
|
-
extent_date_min, extent_date_max = self.get_datetime_extent()
|
|
978
|
-
|
|
979
|
-
return list(
|
|
980
|
-
range(
|
|
981
|
-
extent_date_min.month,
|
|
982
|
-
(extent_date_max - relativedelta(days=1)).month + 1,
|
|
983
|
-
)
|
|
984
|
-
)
|
|
985
|
-
|
|
986
|
-
def get_stac_days_list(self, **_: Any) -> List[int]:
|
|
987
|
-
"""Get catalog available days list
|
|
988
|
-
|
|
989
|
-
:returns: Days list
|
|
990
|
-
"""
|
|
991
|
-
extent_date_min, extent_date_max = self.get_datetime_extent()
|
|
992
|
-
|
|
993
|
-
return list(
|
|
994
|
-
range(
|
|
995
|
-
extent_date_min.day, (extent_date_max - relativedelta(days=1)).day + 1
|
|
996
|
-
)
|
|
997
|
-
)
|
|
998
|
-
|
|
999
|
-
def set_stac_year_by_id(self, year: str, **_: Any) -> Dict[str, Any]:
|
|
1000
|
-
"""Updates and returns catalog with given year
|
|
1001
|
-
|
|
1002
|
-
:param year: Year number
|
|
1003
|
-
:returns: Updated catalog
|
|
1004
|
-
"""
|
|
1005
|
-
extent_date_min, extent_date_max = self.get_datetime_extent()
|
|
1006
|
-
|
|
1007
|
-
datetime_min = max(
|
|
1008
|
-
[extent_date_min, dateutil.parser.parse(f"{year}-01-01T00:00:00Z")]
|
|
1009
|
-
)
|
|
1010
|
-
datetime_max = min(
|
|
1011
|
-
[
|
|
1012
|
-
extent_date_max,
|
|
1013
|
-
dateutil.parser.parse(f"{year}-01-01T00:00:00Z")
|
|
1014
|
-
+ relativedelta(years=1),
|
|
1015
|
-
]
|
|
1016
|
-
)
|
|
1017
|
-
|
|
1018
|
-
catalog_model = deepcopy(self.stac_config["catalogs"]["year"]["model"])
|
|
1019
|
-
|
|
1020
|
-
parsed_dict = self.set_stac_date(datetime_min, datetime_max, catalog_model)
|
|
1021
|
-
|
|
1022
|
-
return parsed_dict
|
|
1023
|
-
|
|
1024
|
-
def set_stac_month_by_id(self, month: str, **_: Any) -> Dict[str, Any]:
|
|
1025
|
-
"""Updates and returns catalog with given month
|
|
1026
|
-
|
|
1027
|
-
:param month: Month number
|
|
1028
|
-
:returns: Updated catalog
|
|
1029
|
-
"""
|
|
1030
|
-
extent_date_min, extent_date_max = self.get_datetime_extent()
|
|
1031
|
-
year = extent_date_min.year
|
|
1032
|
-
|
|
1033
|
-
datetime_min = max(
|
|
1034
|
-
[
|
|
1035
|
-
extent_date_min,
|
|
1036
|
-
dateutil.parser.parse(f"{year}-{month}-01T00:00:00Z"),
|
|
1037
|
-
]
|
|
1038
|
-
)
|
|
1039
|
-
datetime_max = min(
|
|
1040
|
-
[
|
|
1041
|
-
extent_date_max,
|
|
1042
|
-
dateutil.parser.parse(f"{year}-{month}-01T00:00:00Z")
|
|
1043
|
-
+ relativedelta(months=1),
|
|
1044
|
-
]
|
|
1045
|
-
)
|
|
1046
|
-
|
|
1047
|
-
catalog_model = deepcopy(self.stac_config["catalogs"]["month"]["model"])
|
|
1048
|
-
|
|
1049
|
-
parsed_dict = self.set_stac_date(datetime_min, datetime_max, catalog_model)
|
|
1050
|
-
|
|
1051
|
-
return parsed_dict
|
|
1052
|
-
|
|
1053
|
-
def set_stac_day_by_id(self, day: str, **_: Any) -> Dict[str, Any]:
|
|
1054
|
-
"""Updates and returns catalog with given day
|
|
1055
|
-
|
|
1056
|
-
:param day: Day number
|
|
1057
|
-
:returns: Updated catalog
|
|
1058
|
-
"""
|
|
1059
|
-
extent_date_min, extent_date_max = self.get_datetime_extent()
|
|
1060
|
-
year = extent_date_min.year
|
|
1061
|
-
month = extent_date_min.month
|
|
1062
|
-
|
|
1063
|
-
datetime_min = max(
|
|
1064
|
-
[
|
|
1065
|
-
extent_date_min,
|
|
1066
|
-
dateutil.parser.parse(f"{year}-{month}-{day}T00:00:00Z"),
|
|
1067
|
-
]
|
|
1068
|
-
)
|
|
1069
|
-
datetime_max = min(
|
|
1070
|
-
[
|
|
1071
|
-
extent_date_max,
|
|
1072
|
-
dateutil.parser.parse(f"{year}-{month}-{day}T00:00:00Z")
|
|
1073
|
-
+ relativedelta(days=1),
|
|
1074
|
-
]
|
|
1075
|
-
)
|
|
1076
|
-
|
|
1077
|
-
catalog_model = deepcopy(self.stac_config["catalogs"]["day"]["model"])
|
|
1078
|
-
|
|
1079
|
-
parsed_dict = self.set_stac_date(datetime_min, datetime_max, catalog_model)
|
|
1080
|
-
|
|
1081
|
-
return parsed_dict
|
|
1082
|
-
|
|
1083
|
-
def get_datetime_extent(self) -> Tuple[datetime, datetime]:
|
|
1084
|
-
"""Returns catalog temporal extent as datetime objs
|
|
1085
|
-
|
|
1086
|
-
:returns: Start & stop dates
|
|
1087
|
-
"""
|
|
1088
|
-
extent_date_min = dateutil.parser.parse(DEFAULT_MISSION_START_DATE).replace(
|
|
1089
|
-
tzinfo=tz.UTC
|
|
1090
|
-
)
|
|
1091
|
-
extent_date_max = datetime.now(timezone.utc).replace(tzinfo=tz.UTC)
|
|
1092
|
-
for interval in self.data["extent"]["temporal"]["interval"]:
|
|
1093
|
-
extent_date_min_str, extent_date_max_str = interval
|
|
1094
|
-
# date min
|
|
1095
|
-
if extent_date_min_str:
|
|
1096
|
-
extent_date_min = max(
|
|
1097
|
-
extent_date_min, dateutil.parser.parse(extent_date_min_str)
|
|
1098
|
-
)
|
|
1099
|
-
# date max
|
|
1100
|
-
if extent_date_max_str:
|
|
1101
|
-
extent_date_max = min(
|
|
1102
|
-
extent_date_max, dateutil.parser.parse(extent_date_max_str)
|
|
1103
|
-
)
|
|
1104
|
-
|
|
1105
|
-
return (
|
|
1106
|
-
extent_date_min.replace(tzinfo=tz.UTC),
|
|
1107
|
-
extent_date_max.replace(tzinfo=tz.UTC),
|
|
1108
|
-
)
|
|
1109
|
-
|
|
1110
|
-
def set_stac_date(
|
|
1111
|
-
self,
|
|
1112
|
-
datetime_min: datetime,
|
|
1113
|
-
datetime_max: datetime,
|
|
1114
|
-
catalog_model: Dict[str, Any],
|
|
1115
|
-
):
|
|
1116
|
-
"""Updates catalog data using given dates
|
|
1117
|
-
|
|
1118
|
-
:param datetime_min: Date min of interval
|
|
1119
|
-
:param datetime_max: Date max of interval
|
|
1120
|
-
:param catalog_model: Catalog model to use, from yml stac_config[catalogs]
|
|
1121
|
-
:returns: Updated catalog
|
|
1122
|
-
"""
|
|
1123
|
-
# parse f-strings
|
|
1124
|
-
format_args = deepcopy(self.stac_config)
|
|
1125
|
-
format_args["catalog"] = defaultdict(str, **self.data)
|
|
1126
|
-
format_args["date"] = defaultdict(
|
|
1127
|
-
str,
|
|
1128
|
-
{
|
|
1129
|
-
"year": datetime_min.year,
|
|
1130
|
-
"month": datetime_min.month,
|
|
1131
|
-
"day": datetime_min.day,
|
|
1132
|
-
"min": datetime_min.isoformat().replace("+00:00", "Z"),
|
|
1133
|
-
"max": datetime_max.isoformat().replace("+00:00", "Z"),
|
|
1134
|
-
},
|
|
1135
|
-
)
|
|
1136
|
-
parsed_dict: Dict[str, Any] = format_dict_items(catalog_model, **format_args)
|
|
1137
|
-
|
|
1138
|
-
self.update_data(parsed_dict)
|
|
1139
|
-
|
|
1140
|
-
# update search args
|
|
1141
|
-
self.search_args.update(
|
|
1142
|
-
{
|
|
1143
|
-
"start": datetime_min.isoformat().replace("+00:00", "Z"),
|
|
1144
|
-
"end": datetime_max.isoformat().replace("+00:00", "Z"),
|
|
1145
|
-
}
|
|
1146
|
-
)
|
|
1147
|
-
return parsed_dict
|
|
1148
|
-
|
|
1149
|
-
# get / set cloud_cover filter --------------------------------------------
|
|
1150
|
-
|
|
1151
|
-
def get_stac_cloud_covers_list(self, **_: Any) -> List[int]:
|
|
1152
|
-
"""Get cloud_cover list
|
|
1153
|
-
|
|
1154
|
-
:returns: cloud_cover list
|
|
1155
|
-
"""
|
|
1156
|
-
return list(range(0, 101, 10))
|
|
1157
|
-
|
|
1158
|
-
def set_stac_cloud_cover_by_id(self, cloud_cover: str, **_: Any) -> Dict[str, Any]:
|
|
1159
|
-
"""Updates and returns catalog with given max cloud_cover
|
|
1160
|
-
|
|
1161
|
-
:param cloud_cover: Cloud_cover number
|
|
1162
|
-
:returns: Updated catalog
|
|
1163
|
-
"""
|
|
1164
|
-
cat_model = deepcopy(self.stac_config["catalogs"]["cloud_cover"]["model"])
|
|
1165
|
-
# parse f-strings
|
|
1166
|
-
format_args = deepcopy(self.stac_config)
|
|
1167
|
-
format_args["catalog"] = defaultdict(str, **self.data)
|
|
1168
|
-
format_args["cloud_cover"] = cloud_cover
|
|
1169
|
-
parsed_dict: Dict[str, Any] = format_dict_items(cat_model, **format_args)
|
|
1170
|
-
|
|
1171
|
-
self.update_data(parsed_dict)
|
|
1172
|
-
|
|
1173
|
-
# update search args
|
|
1174
|
-
self.search_args.update({"cloudCover": cloud_cover})
|
|
1175
|
-
|
|
1176
|
-
return parsed_dict
|
|
1177
|
-
|
|
1178
|
-
# get / set locations filter ----------------------------------------------
|
|
1179
|
-
|
|
1180
|
-
def get_stac_location_list(self, catalog_name: str) -> List[str]:
|
|
1181
|
-
"""Get locations list using stac_conf & locations_config
|
|
1182
|
-
|
|
1183
|
-
:param catalog_name: Catalog/location name
|
|
1184
|
-
:returns: Locations list
|
|
1185
|
-
"""
|
|
1186
|
-
|
|
1187
|
-
if catalog_name not in self.stac_config["catalogs"]:
|
|
1188
|
-
logger.warning("no entry found for %s in location_config", catalog_name)
|
|
1189
|
-
return []
|
|
1190
|
-
location_config = self.stac_config["catalogs"][catalog_name]
|
|
1191
|
-
|
|
1192
|
-
for k in ["path", "attr"]:
|
|
1193
|
-
if k not in location_config.keys():
|
|
1194
|
-
logger.warning(
|
|
1195
|
-
"no %s key found for %s in location_config", k, catalog_name
|
|
1196
|
-
)
|
|
1197
|
-
return []
|
|
1198
|
-
path = location_config["path"]
|
|
1199
|
-
attr = location_config["attr"]
|
|
1200
|
-
|
|
1201
|
-
with shapefile.Reader(path) as shp:
|
|
1202
|
-
countries_list: List[str] = [rec[attr] for rec in shp.records()] # type: ignore
|
|
1203
|
-
|
|
1204
|
-
# remove duplicates
|
|
1205
|
-
countries_list = list(set(countries_list))
|
|
1206
|
-
|
|
1207
|
-
countries_list.sort()
|
|
1208
|
-
|
|
1209
|
-
return countries_list
|
|
1210
|
-
|
|
1211
|
-
def set_stac_location_by_id(
|
|
1212
|
-
self, location: str, catalog_name: str
|
|
1213
|
-
) -> Dict[str, Any]:
|
|
1214
|
-
"""Updates and returns catalog with given location
|
|
1215
|
-
|
|
1216
|
-
:param location: Feature attribute value for shp filtering
|
|
1217
|
-
:param catalog_name: Catalog/location name
|
|
1218
|
-
:returns: Updated catalog
|
|
1219
|
-
"""
|
|
1220
|
-
location_list_cat_key = catalog_name + "_list"
|
|
1221
|
-
|
|
1222
|
-
if location_list_cat_key not in self.stac_config["catalogs"]:
|
|
1223
|
-
logger.warning(
|
|
1224
|
-
"no entry found for %s's list in location_config", catalog_name
|
|
1225
|
-
)
|
|
1226
|
-
return {}
|
|
1227
|
-
location_config = self.stac_config["catalogs"][location_list_cat_key]
|
|
1228
|
-
|
|
1229
|
-
for k in ["path", "attr"]:
|
|
1230
|
-
if k not in location_config.keys():
|
|
1231
|
-
logger.warning(
|
|
1232
|
-
"no %s key found for %s's list in location_config", k, catalog_name
|
|
1233
|
-
)
|
|
1234
|
-
return {}
|
|
1235
|
-
path = location_config["path"]
|
|
1236
|
-
attr = location_config["attr"]
|
|
1237
|
-
|
|
1238
|
-
with shapefile.Reader(path) as shp:
|
|
1239
|
-
geom_hits = [
|
|
1240
|
-
shape(shaperec.shape)
|
|
1241
|
-
for shaperec in shp.shapeRecords()
|
|
1242
|
-
if shaperec.record.as_dict().get(attr, None) == location
|
|
1243
|
-
]
|
|
1244
|
-
|
|
1245
|
-
if not geom_hits:
|
|
1246
|
-
logger.warning(
|
|
1247
|
-
"no feature found in %s matching %s=%s", path, attr, location
|
|
1248
|
-
)
|
|
1249
|
-
return {}
|
|
1250
|
-
|
|
1251
|
-
geom = cast(BaseGeometry, unary_union(geom_hits))
|
|
1252
|
-
|
|
1253
|
-
cat_model = deepcopy(self.stac_config["catalogs"]["country"]["model"])
|
|
1254
|
-
# parse f-strings
|
|
1255
|
-
format_args = deepcopy(self.stac_config)
|
|
1256
|
-
format_args["catalog"] = defaultdict(str, **self.data)
|
|
1257
|
-
format_args["feature"] = defaultdict(str, {"geometry": geom, "id": location})
|
|
1258
|
-
parsed_dict: Dict[str, Any] = format_dict_items(cat_model, **format_args)
|
|
1259
|
-
|
|
1260
|
-
self.update_data(parsed_dict)
|
|
1261
|
-
|
|
1262
|
-
# update search args
|
|
1263
|
-
self.search_args.update({"geom": geom})
|
|
1264
|
-
|
|
1265
|
-
return parsed_dict
|
|
1266
|
-
|
|
1267
|
-
def build_locations_config(self) -> Dict[str, str]:
|
|
1268
|
-
"""Build locations config from stac_conf[locations_catalogs] & eodag_api.locations_config
|
|
1269
|
-
|
|
1270
|
-
:returns: Locations configuration dict
|
|
1271
|
-
"""
|
|
1272
|
-
user_config_locations_list = self.eodag_api.locations_config
|
|
1273
|
-
|
|
1274
|
-
locations_config_model = deepcopy(self.stac_config["locations_catalogs"])
|
|
1275
|
-
|
|
1276
|
-
locations_config: Dict[str, str] = {}
|
|
1277
|
-
for loc in user_config_locations_list:
|
|
1278
|
-
# parse jsonpath
|
|
1279
|
-
parsed = jsonpath_parse_dict_items(
|
|
1280
|
-
locations_config_model, {"shp_location": loc}
|
|
1281
|
-
)
|
|
1282
|
-
|
|
1283
|
-
# set default child/parent for this location
|
|
1284
|
-
parsed["location"]["parent_key"] = f"{loc['name']}_list"
|
|
1285
|
-
|
|
1286
|
-
locations_config[f"{loc['name']}_list"] = parsed["locations_list"]
|
|
1287
|
-
locations_config[loc["name"]] = parsed["location"]
|
|
1288
|
-
|
|
1289
|
-
return locations_config
|
|
1290
|
-
|
|
1291
|
-
def __build_stac_catalog(self, catalogs: Optional[List[str]] = None) -> StacCatalog:
|
|
1292
|
-
"""Build nested catalog from catalag list
|
|
1293
|
-
|
|
1294
|
-
:param catalogs: (optional) Catalogs list
|
|
1295
|
-
:returns: This catalog obj
|
|
1296
|
-
"""
|
|
1297
|
-
settings = Settings.from_environment()
|
|
1298
|
-
|
|
1299
|
-
# update conf with user shp locations
|
|
1300
|
-
locations_config = self.build_locations_config()
|
|
1301
|
-
|
|
1302
|
-
self.stac_config["catalogs"] = {
|
|
1303
|
-
**deepcopy(self.stac_config["catalogs"]),
|
|
1304
|
-
**locations_config,
|
|
1305
|
-
}
|
|
1306
|
-
|
|
1307
|
-
if not catalogs:
|
|
1308
|
-
# Build root catalog combined with landing page
|
|
1309
|
-
self.__update_data_from_catalog_config(
|
|
1310
|
-
{
|
|
1311
|
-
"model": {
|
|
1312
|
-
**deepcopy(self.stac_config["landing_page"]),
|
|
1313
|
-
**{
|
|
1314
|
-
"provider": self.provider,
|
|
1315
|
-
"id": settings.stac_api_landing_id,
|
|
1316
|
-
"title": settings.stac_api_title,
|
|
1317
|
-
"description": settings.stac_api_description,
|
|
1318
|
-
},
|
|
1319
|
-
}
|
|
1320
|
-
}
|
|
1321
|
-
)
|
|
1322
|
-
|
|
1323
|
-
# build children : product_types
|
|
1324
|
-
product_types_list = [
|
|
1325
|
-
pt
|
|
1326
|
-
for pt in self.eodag_api.list_product_types(
|
|
1327
|
-
provider=self.provider, fetch_providers=False
|
|
1328
|
-
)
|
|
1329
|
-
]
|
|
1330
|
-
self.set_children(
|
|
1331
|
-
[
|
|
1332
|
-
{
|
|
1333
|
-
"rel": "child",
|
|
1334
|
-
"href": urljoin(
|
|
1335
|
-
self.url, f"{STAC_CATALOGS_PREFIX}/{product_type['ID']}"
|
|
1336
|
-
),
|
|
1337
|
-
"title": product_type["title"],
|
|
1338
|
-
}
|
|
1339
|
-
for product_type in product_types_list
|
|
1340
|
-
]
|
|
1341
|
-
)
|
|
1342
|
-
return self
|
|
1343
|
-
|
|
1344
|
-
# use product_types_list as base for building nested catalogs
|
|
1345
|
-
self.__update_data_from_catalog_config(
|
|
1346
|
-
deepcopy(self.stac_config["catalogs"]["product_types_list"])
|
|
1347
|
-
)
|
|
1348
|
-
|
|
1349
|
-
for idx, cat in enumerate(catalogs):
|
|
1350
|
-
if idx % 2 == 0:
|
|
1351
|
-
# even: cat is a filtering value ----------------------------------
|
|
1352
|
-
cat_data_name = self.catalog_config["child_key"]
|
|
1353
|
-
cat_data_value = cat
|
|
1354
|
-
|
|
1355
|
-
# update data
|
|
1356
|
-
cat_data_name_dict = self.stac_config["catalogs"][cat_data_name]
|
|
1357
|
-
set_data_method_name = (
|
|
1358
|
-
f"set_stac_{cat_data_name}_by_id"
|
|
1359
|
-
if "catalog_type" not in cat_data_name_dict.keys()
|
|
1360
|
-
else f"set_stac_{cat_data_name_dict['catalog_type']}_by_id"
|
|
1361
|
-
)
|
|
1362
|
-
set_data_method = getattr(self, set_data_method_name)
|
|
1363
|
-
set_data_method(cat_data_value, catalog_name=cat_data_name)
|
|
1364
|
-
|
|
1365
|
-
if idx == len(catalogs) - 1:
|
|
1366
|
-
# build children : remaining filtering keys
|
|
1367
|
-
remaining_catalogs_list = [
|
|
1368
|
-
c
|
|
1369
|
-
for c in self.stac_config["catalogs"].keys()
|
|
1370
|
-
# keep filters not used yet AND
|
|
1371
|
-
if self.stac_config["catalogs"][c]["model"]["id"]
|
|
1372
|
-
not in catalogs
|
|
1373
|
-
and (
|
|
1374
|
-
# filters with no parent_key constraint (no key, or key=None) OR
|
|
1375
|
-
"parent_key" not in self.stac_config["catalogs"][c]
|
|
1376
|
-
or not self.stac_config["catalogs"][c]["parent_key"]
|
|
1377
|
-
# filters matching parent_key constraint
|
|
1378
|
-
or self.stac_config["catalogs"][c]["parent_key"]
|
|
1379
|
-
== cat_data_name
|
|
1380
|
-
)
|
|
1381
|
-
# AND filters that match parent attr constraint (locations)
|
|
1382
|
-
and (
|
|
1383
|
-
"parent" not in self.stac_config["catalogs"][c]
|
|
1384
|
-
or not self.stac_config["catalogs"][c]["parent"]["key"]
|
|
1385
|
-
or (
|
|
1386
|
-
self.stac_config["catalogs"][c]["parent"]["key"]
|
|
1387
|
-
== cat_data_name
|
|
1388
|
-
and self.stac_config["catalogs"][c]["parent"]["attr"]
|
|
1389
|
-
== cat_data_value
|
|
1390
|
-
)
|
|
1391
|
-
)
|
|
1392
|
-
]
|
|
1393
|
-
|
|
1394
|
-
self.set_children(
|
|
1395
|
-
[
|
|
1396
|
-
{
|
|
1397
|
-
"rel": "child",
|
|
1398
|
-
"href": self.url
|
|
1399
|
-
+ "/"
|
|
1400
|
-
+ self.stac_config["catalogs"][c]["model"]["id"],
|
|
1401
|
-
"title": str(
|
|
1402
|
-
self.stac_config["catalogs"][c]["model"]["id"]
|
|
1403
|
-
),
|
|
1404
|
-
}
|
|
1405
|
-
for c in remaining_catalogs_list
|
|
1406
|
-
]
|
|
1407
|
-
+ [
|
|
1408
|
-
{
|
|
1409
|
-
"rel": "items",
|
|
1410
|
-
"href": self.url + "/items",
|
|
1411
|
-
"title": "items",
|
|
1412
|
-
}
|
|
1413
|
-
]
|
|
1414
|
-
)
|
|
1415
|
-
|
|
1416
|
-
else:
|
|
1417
|
-
# odd: cat is a filtering key -------------------------------------
|
|
1418
|
-
try:
|
|
1419
|
-
cat_key = [
|
|
1420
|
-
c
|
|
1421
|
-
for c in self.stac_config["catalogs"].keys()
|
|
1422
|
-
if self.stac_config["catalogs"][c]["model"]["id"] == cat
|
|
1423
|
-
][0]
|
|
1424
|
-
except IndexError as e:
|
|
1425
|
-
raise ValidationError(
|
|
1426
|
-
f"Bad settings for {cat} in stac_config catalogs"
|
|
1427
|
-
) from e
|
|
1428
|
-
cat_config = deepcopy(self.stac_config["catalogs"][cat_key])
|
|
1429
|
-
# update data
|
|
1430
|
-
self.__update_data_from_catalog_config(cat_config)
|
|
1431
|
-
|
|
1432
|
-
# get filtering values list
|
|
1433
|
-
get_data_method_name = (
|
|
1434
|
-
f"get_stac_{cat_key}"
|
|
1435
|
-
if "catalog_type"
|
|
1436
|
-
not in self.stac_config["catalogs"][cat_key].keys()
|
|
1437
|
-
else f"get_stac_{self.stac_config['catalogs'][cat_key]['catalog_type']}"
|
|
1438
|
-
)
|
|
1439
|
-
get_data_method = getattr(self, get_data_method_name)
|
|
1440
|
-
cat_data_list = get_data_method(catalog_name=cat_key)
|
|
1441
|
-
|
|
1442
|
-
if idx == len(catalogs) - 1:
|
|
1443
|
-
# filtering values list as children (do not include items)
|
|
1444
|
-
self.set_children(
|
|
1445
|
-
[
|
|
1446
|
-
{
|
|
1447
|
-
"rel": "child",
|
|
1448
|
-
"href": self.url + "/" + str(filtering_data),
|
|
1449
|
-
"title": str(filtering_data),
|
|
1450
|
-
}
|
|
1451
|
-
for filtering_data in cat_data_list
|
|
1452
|
-
]
|
|
1453
|
-
)
|
|
1454
|
-
|
|
1455
|
-
return self
|