eodag 3.0.0b3__py3-none-any.whl → 3.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- eodag/api/core.py +347 -247
- eodag/api/product/_assets.py +44 -15
- eodag/api/product/_product.py +58 -47
- eodag/api/product/drivers/__init__.py +81 -4
- eodag/api/product/drivers/base.py +65 -4
- eodag/api/product/drivers/generic.py +65 -0
- eodag/api/product/drivers/sentinel1.py +97 -0
- eodag/api/product/drivers/sentinel2.py +95 -0
- eodag/api/product/metadata_mapping.py +129 -93
- eodag/api/search_result.py +28 -12
- eodag/cli.py +61 -24
- eodag/config.py +457 -167
- eodag/plugins/apis/base.py +10 -4
- eodag/plugins/apis/ecmwf.py +53 -23
- eodag/plugins/apis/usgs.py +41 -17
- eodag/plugins/authentication/aws_auth.py +30 -18
- eodag/plugins/authentication/base.py +14 -3
- eodag/plugins/authentication/generic.py +14 -3
- eodag/plugins/authentication/header.py +14 -6
- eodag/plugins/authentication/keycloak.py +44 -25
- eodag/plugins/authentication/oauth.py +18 -4
- eodag/plugins/authentication/openid_connect.py +192 -171
- eodag/plugins/authentication/qsauth.py +12 -4
- eodag/plugins/authentication/sas_auth.py +22 -5
- eodag/plugins/authentication/token.py +95 -17
- eodag/plugins/authentication/token_exchange.py +19 -19
- eodag/plugins/base.py +4 -4
- eodag/plugins/crunch/base.py +8 -5
- eodag/plugins/crunch/filter_date.py +9 -6
- eodag/plugins/crunch/filter_latest_intersect.py +9 -8
- eodag/plugins/crunch/filter_latest_tpl_name.py +8 -8
- eodag/plugins/crunch/filter_overlap.py +9 -11
- eodag/plugins/crunch/filter_property.py +10 -10
- eodag/plugins/download/aws.py +181 -105
- eodag/plugins/download/base.py +49 -67
- eodag/plugins/download/creodias_s3.py +40 -2
- eodag/plugins/download/http.py +247 -223
- eodag/plugins/download/s3rest.py +29 -28
- eodag/plugins/manager.py +176 -41
- eodag/plugins/search/__init__.py +6 -5
- eodag/plugins/search/base.py +123 -60
- eodag/plugins/search/build_search_result.py +1046 -355
- eodag/plugins/search/cop_marine.py +132 -39
- eodag/plugins/search/creodias_s3.py +19 -68
- eodag/plugins/search/csw.py +48 -8
- eodag/plugins/search/data_request_search.py +124 -23
- eodag/plugins/search/qssearch.py +531 -310
- eodag/plugins/search/stac_list_assets.py +85 -0
- eodag/plugins/search/static_stac_search.py +23 -24
- eodag/resources/ext_product_types.json +1 -1
- eodag/resources/product_types.yml +1295 -355
- eodag/resources/providers.yml +1819 -3010
- eodag/resources/stac.yml +3 -163
- eodag/resources/stac_api.yml +2 -2
- eodag/resources/user_conf_template.yml +115 -99
- eodag/rest/cache.py +2 -2
- eodag/rest/config.py +3 -4
- eodag/rest/constants.py +0 -1
- eodag/rest/core.py +157 -117
- eodag/rest/errors.py +181 -0
- eodag/rest/server.py +57 -339
- eodag/rest/stac.py +133 -581
- eodag/rest/types/collections_search.py +3 -3
- eodag/rest/types/eodag_search.py +41 -30
- eodag/rest/types/queryables.py +42 -32
- eodag/rest/types/stac_search.py +15 -16
- eodag/rest/utils/__init__.py +14 -21
- eodag/rest/utils/cql_evaluate.py +6 -6
- eodag/rest/utils/rfc3339.py +2 -2
- eodag/types/__init__.py +153 -32
- eodag/types/bbox.py +2 -2
- eodag/types/download_args.py +4 -4
- eodag/types/queryables.py +183 -73
- eodag/types/search_args.py +6 -6
- eodag/types/whoosh.py +127 -3
- eodag/utils/__init__.py +228 -106
- eodag/utils/exceptions.py +47 -26
- eodag/utils/import_system.py +2 -2
- eodag/utils/logging.py +37 -77
- eodag/utils/repr.py +65 -6
- eodag/utils/requests.py +13 -15
- eodag/utils/rest.py +2 -2
- eodag/utils/s3.py +231 -0
- eodag/utils/stac_reader.py +11 -11
- {eodag-3.0.0b3.dist-info → eodag-3.1.0.dist-info}/METADATA +81 -81
- eodag-3.1.0.dist-info/RECORD +113 -0
- {eodag-3.0.0b3.dist-info → eodag-3.1.0.dist-info}/WHEEL +1 -1
- {eodag-3.0.0b3.dist-info → eodag-3.1.0.dist-info}/entry_points.txt +5 -2
- eodag/resources/constraints/climate-dt.json +0 -13
- eodag/resources/constraints/extremes-dt.json +0 -8
- eodag/utils/constraints.py +0 -244
- eodag-3.0.0b3.dist-info/RECORD +0 -110
- {eodag-3.0.0b3.dist-info → eodag-3.1.0.dist-info}/LICENSE +0 -0
- {eodag-3.0.0b3.dist-info → eodag-3.1.0.dist-info}/top_level.txt +0 -0
eodag/rest/stac.py
CHANGED
|
@@ -21,18 +21,19 @@ import logging
|
|
|
21
21
|
import os
|
|
22
22
|
from collections import defaultdict
|
|
23
23
|
from datetime import datetime, timezone
|
|
24
|
-
from typing import TYPE_CHECKING, Any,
|
|
25
|
-
from urllib.parse import
|
|
24
|
+
from typing import TYPE_CHECKING, Any, Optional
|
|
25
|
+
from urllib.parse import (
|
|
26
|
+
parse_qs,
|
|
27
|
+
quote,
|
|
28
|
+
urlencode,
|
|
29
|
+
urlparse,
|
|
30
|
+
urlsplit,
|
|
31
|
+
urlunparse,
|
|
32
|
+
urlunsplit,
|
|
33
|
+
)
|
|
26
34
|
|
|
27
|
-
import dateutil.parser
|
|
28
35
|
import geojson
|
|
29
|
-
import shapefile
|
|
30
|
-
from dateutil import tz
|
|
31
|
-
from dateutil.relativedelta import relativedelta
|
|
32
36
|
from jsonpath_ng.jsonpath import Child
|
|
33
|
-
from shapely.geometry import shape
|
|
34
|
-
from shapely.geometry.base import BaseGeometry
|
|
35
|
-
from shapely.ops import unary_union
|
|
36
37
|
|
|
37
38
|
from eodag.api.product.metadata_mapping import (
|
|
38
39
|
DEFAULT_METADATA_MAPPING,
|
|
@@ -42,7 +43,6 @@ from eodag.api.product.metadata_mapping import (
|
|
|
42
43
|
from eodag.rest.config import Settings
|
|
43
44
|
from eodag.rest.utils.rfc3339 import str_to_interval
|
|
44
45
|
from eodag.utils import (
|
|
45
|
-
DEFAULT_MISSION_START_DATE,
|
|
46
46
|
deepcopy,
|
|
47
47
|
dict_items_recursive_apply,
|
|
48
48
|
format_dict_items,
|
|
@@ -50,14 +50,12 @@ from eodag.utils import (
|
|
|
50
50
|
jsonpath_parse_dict_items,
|
|
51
51
|
string_to_jsonpath,
|
|
52
52
|
update_nested_dict,
|
|
53
|
-
urljoin,
|
|
54
53
|
)
|
|
55
54
|
from eodag.utils.exceptions import (
|
|
56
55
|
NoMatchingProductType,
|
|
57
56
|
NotAvailableError,
|
|
58
57
|
RequestError,
|
|
59
58
|
TimeOutError,
|
|
60
|
-
ValidationError,
|
|
61
59
|
)
|
|
62
60
|
from eodag.utils.requests import fetch_json
|
|
63
61
|
|
|
@@ -69,8 +67,6 @@ if TYPE_CHECKING:
|
|
|
69
67
|
|
|
70
68
|
logger = logging.getLogger("eodag.rest.stac")
|
|
71
69
|
|
|
72
|
-
STAC_CATALOGS_PREFIX = "catalogs"
|
|
73
|
-
|
|
74
70
|
# fields not to put in item properties
|
|
75
71
|
COLLECTION_PROPERTIES = [
|
|
76
72
|
"abstract",
|
|
@@ -86,6 +82,8 @@ COLLECTION_PROPERTIES = [
|
|
|
86
82
|
"missionEndDate",
|
|
87
83
|
"keywords",
|
|
88
84
|
"stacCollection",
|
|
85
|
+
"alias",
|
|
86
|
+
"productType",
|
|
89
87
|
]
|
|
90
88
|
IGNORED_ITEM_PROPERTIES = [
|
|
91
89
|
"_id",
|
|
@@ -99,9 +97,17 @@ IGNORED_ITEM_PROPERTIES = [
|
|
|
99
97
|
"qs",
|
|
100
98
|
"defaultGeometry",
|
|
101
99
|
"_date",
|
|
100
|
+
"productType",
|
|
102
101
|
]
|
|
103
102
|
|
|
104
103
|
|
|
104
|
+
def _quote_url_path(url: str) -> str:
|
|
105
|
+
parsed = urlsplit(url)
|
|
106
|
+
path = quote(parsed.path)
|
|
107
|
+
components = (parsed.scheme, parsed.netloc, path, parsed.query, parsed.fragment)
|
|
108
|
+
return urlunsplit(components)
|
|
109
|
+
|
|
110
|
+
|
|
105
111
|
class StacCommon:
|
|
106
112
|
"""Stac common object
|
|
107
113
|
|
|
@@ -115,7 +121,7 @@ class StacCommon:
|
|
|
115
121
|
def __init__(
|
|
116
122
|
self,
|
|
117
123
|
url: str,
|
|
118
|
-
stac_config:
|
|
124
|
+
stac_config: dict[str, Any],
|
|
119
125
|
provider: Optional[str],
|
|
120
126
|
eodag_api: EODataAccessGateway,
|
|
121
127
|
root: str = "/",
|
|
@@ -126,9 +132,9 @@ class StacCommon:
|
|
|
126
132
|
self.eodag_api = eodag_api
|
|
127
133
|
self.root = root.rstrip("/") if len(root) > 1 else root
|
|
128
134
|
|
|
129
|
-
self.data:
|
|
135
|
+
self.data: dict[str, Any] = {}
|
|
130
136
|
|
|
131
|
-
def update_data(self, data:
|
|
137
|
+
def update_data(self, data: dict[str, Any]) -> None:
|
|
132
138
|
"""Updates data using given input STAC dict data
|
|
133
139
|
|
|
134
140
|
:param data: Catalog data (parsed STAC dict)
|
|
@@ -162,15 +168,15 @@ class StacCommon:
|
|
|
162
168
|
|
|
163
169
|
@staticmethod
|
|
164
170
|
def get_stac_extension(
|
|
165
|
-
url: str, stac_config:
|
|
166
|
-
) ->
|
|
171
|
+
url: str, stac_config: dict[str, Any], extension: str, **kwargs: Any
|
|
172
|
+
) -> dict[str, str]:
|
|
167
173
|
"""Parse STAC extension from config and return as dict
|
|
168
174
|
|
|
169
175
|
:param url: Requested URL
|
|
170
176
|
:param stac_config: STAC configuration from stac.yml conf file
|
|
171
177
|
:param extension: Extension name
|
|
172
178
|
:param kwargs: Additional variables needed for parsing extension
|
|
173
|
-
:returns: STAC extension as
|
|
179
|
+
:returns: STAC extension as dictionary
|
|
174
180
|
"""
|
|
175
181
|
extension_model = deepcopy(stac_config).get("extensions", {}).get(extension, {})
|
|
176
182
|
|
|
@@ -182,7 +188,7 @@ class StacCommon:
|
|
|
182
188
|
}
|
|
183
189
|
return format_dict_items(extension_model, **format_args)
|
|
184
190
|
|
|
185
|
-
def get_provider_dict(self, provider: str) ->
|
|
191
|
+
def get_provider_dict(self, provider: str) -> dict[str, Any]:
|
|
186
192
|
"""Generate STAC provider dict"""
|
|
187
193
|
provider_config = next(
|
|
188
194
|
p
|
|
@@ -211,7 +217,7 @@ class StacItem(StacCommon):
|
|
|
211
217
|
def __init__(
|
|
212
218
|
self,
|
|
213
219
|
url: str,
|
|
214
|
-
stac_config:
|
|
220
|
+
stac_config: dict[str, Any],
|
|
215
221
|
provider: Optional[str],
|
|
216
222
|
eodag_api: EODataAccessGateway,
|
|
217
223
|
root: str = "/",
|
|
@@ -225,8 +231,8 @@ class StacItem(StacCommon):
|
|
|
225
231
|
)
|
|
226
232
|
|
|
227
233
|
def __get_item_list(
|
|
228
|
-
self, search_results: SearchResult, catalog:
|
|
229
|
-
) ->
|
|
234
|
+
self, search_results: SearchResult, catalog: dict[str, Any]
|
|
235
|
+
) -> list[dict[str, Any]]:
|
|
230
236
|
"""Build STAC items list from EODAG search results
|
|
231
237
|
|
|
232
238
|
:param search_results: EODAG search results
|
|
@@ -241,7 +247,7 @@ class StacItem(StacCommon):
|
|
|
241
247
|
)
|
|
242
248
|
|
|
243
249
|
# check if some items need to be converted
|
|
244
|
-
need_conversion:
|
|
250
|
+
need_conversion: dict[str, Any] = {}
|
|
245
251
|
for k, v in item_model["properties"].items():
|
|
246
252
|
if isinstance(v, str):
|
|
247
253
|
conversion, item_model["properties"][k] = get_metadata_path(
|
|
@@ -261,11 +267,11 @@ class StacItem(StacCommon):
|
|
|
261
267
|
]
|
|
262
268
|
ignored_props = COLLECTION_PROPERTIES + item_props + IGNORED_ITEM_PROPERTIES
|
|
263
269
|
|
|
264
|
-
item_list:
|
|
270
|
+
item_list: list[dict[str, Any]] = []
|
|
265
271
|
for product in search_results:
|
|
266
272
|
product_dict = deepcopy(product.__dict__)
|
|
267
273
|
|
|
268
|
-
product_item:
|
|
274
|
+
product_item: dict[str, Any] = jsonpath_parse_dict_items(
|
|
269
275
|
item_model,
|
|
270
276
|
{
|
|
271
277
|
"product": product_dict,
|
|
@@ -342,6 +348,10 @@ class StacItem(StacCommon):
|
|
|
342
348
|
# remove empty properties
|
|
343
349
|
product_item = self.__filter_item_properties_values(product_item)
|
|
344
350
|
|
|
351
|
+
# quote invalid characters in links
|
|
352
|
+
for link in product_item["links"]:
|
|
353
|
+
link["href"] = _quote_url_path(link["href"])
|
|
354
|
+
|
|
345
355
|
# update item link with datacube query-string
|
|
346
356
|
if _dc_qs or self.provider:
|
|
347
357
|
url_parts = urlparse(str(product_item["links"][0]["href"]))
|
|
@@ -363,10 +373,10 @@ class StacItem(StacCommon):
|
|
|
363
373
|
product: EOProduct,
|
|
364
374
|
downloadlink_href: str,
|
|
365
375
|
without_arg_url: str,
|
|
366
|
-
query_dict: Optional[
|
|
376
|
+
query_dict: Optional[dict[str, Any]] = None,
|
|
367
377
|
_dc_qs: Optional[str] = None,
|
|
368
|
-
) ->
|
|
369
|
-
assets:
|
|
378
|
+
) -> dict[str, Any]:
|
|
379
|
+
assets: dict[str, Any] = {}
|
|
370
380
|
settings = Settings.from_environment()
|
|
371
381
|
|
|
372
382
|
if _dc_qs:
|
|
@@ -378,9 +388,12 @@ class StacItem(StacCommon):
|
|
|
378
388
|
origin_href = product.remote_location
|
|
379
389
|
|
|
380
390
|
# update download link with up-to-date query-args
|
|
391
|
+
quoted_href = _quote_url_path(
|
|
392
|
+
downloadlink_href
|
|
393
|
+
) # quote invalid characters in url
|
|
381
394
|
assets["downloadLink"] = {
|
|
382
395
|
"title": "Download link",
|
|
383
|
-
"href":
|
|
396
|
+
"href": quoted_href,
|
|
384
397
|
"type": "application/zip",
|
|
385
398
|
}
|
|
386
399
|
|
|
@@ -424,6 +437,7 @@ class StacItem(StacCommon):
|
|
|
424
437
|
assets[asset_key]["type"] = asset_type
|
|
425
438
|
if origin := assets[asset_key].get("alternate", {}).get("origin"):
|
|
426
439
|
origin["type"] = asset_type
|
|
440
|
+
asset_value["href"] = _quote_url_path(asset_value["href"])
|
|
427
441
|
|
|
428
442
|
if thumbnail_url := product.properties.get(
|
|
429
443
|
"quicklook", product.properties.get("thumbnail", None)
|
|
@@ -441,14 +455,14 @@ class StacItem(StacCommon):
|
|
|
441
455
|
self,
|
|
442
456
|
search_results: SearchResult,
|
|
443
457
|
total: int,
|
|
444
|
-
catalog:
|
|
445
|
-
next_link: Optional[
|
|
446
|
-
) ->
|
|
458
|
+
catalog: dict[str, Any],
|
|
459
|
+
next_link: Optional[dict[str, Any]],
|
|
460
|
+
) -> dict[str, Any]:
|
|
447
461
|
"""Build STAC items from EODAG search results
|
|
448
462
|
|
|
449
463
|
:param search_results: EODAG search results
|
|
450
464
|
:param catalog: STAC catalog dict used for parsing item metadata
|
|
451
|
-
:returns: Items
|
|
465
|
+
:returns: Items dictionary
|
|
452
466
|
"""
|
|
453
467
|
items_model = deepcopy(self.stac_config["items"])
|
|
454
468
|
|
|
@@ -492,8 +506,8 @@ class StacItem(StacCommon):
|
|
|
492
506
|
return self.data
|
|
493
507
|
|
|
494
508
|
def __filter_item_model_properties(
|
|
495
|
-
self, item_model:
|
|
496
|
-
) ->
|
|
509
|
+
self, item_model: dict[str, Any], product_type: str
|
|
510
|
+
) -> dict[str, Any]:
|
|
497
511
|
"""Filter item model depending on product type metadata and its extensions.
|
|
498
512
|
Removes not needed parameters, and adds supplementary ones as
|
|
499
513
|
part of oseo extension.
|
|
@@ -557,13 +571,13 @@ class StacItem(StacCommon):
|
|
|
557
571
|
|
|
558
572
|
return result_item_model
|
|
559
573
|
|
|
560
|
-
def __filter_item_properties_values(self, item:
|
|
574
|
+
def __filter_item_properties_values(self, item: dict[str, Any]) -> dict[str, Any]:
|
|
561
575
|
"""Removes empty properties, unused extensions, and add missing extensions
|
|
562
576
|
|
|
563
577
|
:param item: STAC item data
|
|
564
578
|
:returns: Filtered item model
|
|
565
579
|
"""
|
|
566
|
-
all_extensions_dict:
|
|
580
|
+
all_extensions_dict: dict[str, str] = deepcopy(
|
|
567
581
|
self.stac_config["stac_extensions"]
|
|
568
582
|
)
|
|
569
583
|
# parse f-strings with root
|
|
@@ -588,7 +602,7 @@ class StacItem(StacCommon):
|
|
|
588
602
|
|
|
589
603
|
return item
|
|
590
604
|
|
|
591
|
-
def get_stac_item_from_product(self, product: EOProduct) ->
|
|
605
|
+
def get_stac_item_from_product(self, product: EOProduct) -> dict[str, Any]:
|
|
592
606
|
"""Build STAC item from EODAG product
|
|
593
607
|
|
|
594
608
|
:param product: EODAG product
|
|
@@ -606,7 +620,7 @@ class StacItem(StacCommon):
|
|
|
606
620
|
root=self.root,
|
|
607
621
|
provider=self.provider,
|
|
608
622
|
eodag_api=self.eodag_api,
|
|
609
|
-
|
|
623
|
+
collection=product_type,
|
|
610
624
|
)
|
|
611
625
|
|
|
612
626
|
product_dict = deepcopy(product.__dict__)
|
|
@@ -648,7 +662,7 @@ class StacCollection(StacCommon):
|
|
|
648
662
|
"""
|
|
649
663
|
|
|
650
664
|
# External STAC collections
|
|
651
|
-
ext_stac_collections:
|
|
665
|
+
ext_stac_collections: dict[str, dict[str, Any]] = dict()
|
|
652
666
|
|
|
653
667
|
@classmethod
|
|
654
668
|
def fetch_external_stac_collections(cls, eodag_api: EODataAccessGateway) -> None:
|
|
@@ -677,7 +691,7 @@ class StacCollection(StacCommon):
|
|
|
677
691
|
def __init__(
|
|
678
692
|
self,
|
|
679
693
|
url: str,
|
|
680
|
-
stac_config:
|
|
694
|
+
stac_config: dict[str, Any],
|
|
681
695
|
provider: Optional[str],
|
|
682
696
|
eodag_api: EODataAccessGateway,
|
|
683
697
|
root: str = "/",
|
|
@@ -690,7 +704,7 @@ class StacCollection(StacCommon):
|
|
|
690
704
|
root=root,
|
|
691
705
|
)
|
|
692
706
|
|
|
693
|
-
def __list_product_type_providers(self, product_type:
|
|
707
|
+
def __list_product_type_providers(self, product_type: dict[str, Any]) -> list[str]:
|
|
694
708
|
"""Retrieve a list of providers for a given product type.
|
|
695
709
|
|
|
696
710
|
:param product_type: Dictionary containing information about the product type.
|
|
@@ -707,8 +721,8 @@ class StacCollection(StacCommon):
|
|
|
707
721
|
]
|
|
708
722
|
|
|
709
723
|
def __generate_stac_collection(
|
|
710
|
-
self, collection_model: Any, product_type:
|
|
711
|
-
) ->
|
|
724
|
+
self, collection_model: Any, product_type: dict[str, Any]
|
|
725
|
+
) -> dict[str, Any]:
|
|
712
726
|
"""Generate a STAC collection dictionary for a given product type.
|
|
713
727
|
|
|
714
728
|
:param collection_model: The base model for the STAC collection.
|
|
@@ -717,7 +731,7 @@ class StacCollection(StacCommon):
|
|
|
717
731
|
"""
|
|
718
732
|
providers = self.__list_product_type_providers(product_type)
|
|
719
733
|
|
|
720
|
-
providers_dict:
|
|
734
|
+
providers_dict: dict[str, dict[str, Any]] = {}
|
|
721
735
|
for provider in providers:
|
|
722
736
|
p_dict = self.get_provider_dict(provider)
|
|
723
737
|
providers_dict.setdefault(p_dict["name"], p_dict)
|
|
@@ -744,25 +758,34 @@ class StacCollection(StacCommon):
|
|
|
744
758
|
]
|
|
745
759
|
ext_stac_collection["links"].append(link)
|
|
746
760
|
|
|
761
|
+
# merge "summaries"
|
|
762
|
+
ext_stac_collection["summaries"] = {
|
|
763
|
+
k: v
|
|
764
|
+
for k, v in {
|
|
765
|
+
**ext_stac_collection.get("summaries", {}),
|
|
766
|
+
**product_type_collection["summaries"],
|
|
767
|
+
}.items()
|
|
768
|
+
if v and any(v)
|
|
769
|
+
}
|
|
770
|
+
|
|
747
771
|
# merge "keywords" lists
|
|
748
|
-
|
|
749
|
-
|
|
750
|
-
|
|
751
|
-
|
|
752
|
-
|
|
753
|
-
|
|
754
|
-
+ product_type_collection["keywords"]
|
|
755
|
-
)
|
|
756
|
-
if k is not None
|
|
757
|
-
]
|
|
758
|
-
except TypeError as e:
|
|
759
|
-
logger.warning(
|
|
760
|
-
f"Could not merge keywords from external collection for {product_type['ID']}: {str(e)}"
|
|
761
|
-
)
|
|
762
|
-
logger.debug(
|
|
763
|
-
f"External collection keywords: {str(ext_stac_collection['keywords'])}, ",
|
|
764
|
-
f"Product type keywords: {str(product_type_collection['keywords'])}",
|
|
772
|
+
try:
|
|
773
|
+
ext_stac_collection["keywords"] = [
|
|
774
|
+
k
|
|
775
|
+
for k in set(
|
|
776
|
+
ext_stac_collection.get("keywords", [])
|
|
777
|
+
+ product_type_collection["keywords"]
|
|
765
778
|
)
|
|
779
|
+
if k is not None
|
|
780
|
+
]
|
|
781
|
+
except TypeError as e:
|
|
782
|
+
logger.warning(
|
|
783
|
+
f"Could not merge keywords from external collection for {product_type['ID']}: {str(e)}"
|
|
784
|
+
)
|
|
785
|
+
logger.debug(
|
|
786
|
+
f"External collection keywords: {str(ext_stac_collection.get('keywords'))}, ",
|
|
787
|
+
f"Product type keywords: {str(product_type_collection['keywords'])}",
|
|
788
|
+
)
|
|
766
789
|
|
|
767
790
|
# merge providers
|
|
768
791
|
if "providers" in ext_stac_collection:
|
|
@@ -795,7 +818,7 @@ class StacCollection(StacCommon):
|
|
|
795
818
|
instrument: Optional[str] = None,
|
|
796
819
|
constellation: Optional[str] = None,
|
|
797
820
|
datetime: Optional[str] = None,
|
|
798
|
-
) ->
|
|
821
|
+
) -> list[dict[str, Any]]:
|
|
799
822
|
"""Build STAC collections list
|
|
800
823
|
|
|
801
824
|
:param filters: (optional) Additional filters for collections search
|
|
@@ -830,7 +853,7 @@ class StacCollection(StacCommon):
|
|
|
830
853
|
product_types = all_pt
|
|
831
854
|
|
|
832
855
|
# list product types with all metadata using guessed ids
|
|
833
|
-
collection_list:
|
|
856
|
+
collection_list: list[dict[str, Any]] = []
|
|
834
857
|
for product_type in product_types:
|
|
835
858
|
stac_collection = self.__generate_stac_collection(
|
|
836
859
|
collection_model, product_type
|
|
@@ -848,17 +871,17 @@ class StacCatalog(StacCommon):
|
|
|
848
871
|
:param provider: Chosen provider
|
|
849
872
|
:param eodag_api: EODAG python API instance
|
|
850
873
|
:param root: (optional) API root
|
|
851
|
-
:param
|
|
874
|
+
:param collection: (optional) product type id
|
|
852
875
|
"""
|
|
853
876
|
|
|
854
877
|
def __init__(
|
|
855
878
|
self,
|
|
856
879
|
url: str,
|
|
857
|
-
stac_config:
|
|
880
|
+
stac_config: dict[str, Any],
|
|
858
881
|
provider: Optional[str],
|
|
859
882
|
eodag_api: EODataAccessGateway,
|
|
860
883
|
root: str = "/",
|
|
861
|
-
|
|
884
|
+
collection: Optional[str] = None,
|
|
862
885
|
) -> None:
|
|
863
886
|
super(StacCatalog, self).__init__(
|
|
864
887
|
url=url,
|
|
@@ -870,8 +893,8 @@ class StacCatalog(StacCommon):
|
|
|
870
893
|
self.data = {}
|
|
871
894
|
|
|
872
895
|
self.shp_location_config = eodag_api.locations_config
|
|
873
|
-
self.search_args:
|
|
874
|
-
self.children:
|
|
896
|
+
self.search_args: dict[str, Any] = {}
|
|
897
|
+
self.children: list[dict[str, Any]] = []
|
|
875
898
|
|
|
876
899
|
self.catalog_config = deepcopy(stac_config["catalog"])
|
|
877
900
|
|
|
@@ -885,9 +908,9 @@ class StacCatalog(StacCommon):
|
|
|
885
908
|
self.data["links"] += self.children
|
|
886
909
|
|
|
887
910
|
# build catalog
|
|
888
|
-
self.__build_stac_catalog(
|
|
911
|
+
self.__build_stac_catalog(collection)
|
|
889
912
|
|
|
890
|
-
def __update_data_from_catalog_config(self, catalog_config:
|
|
913
|
+
def __update_data_from_catalog_config(self, catalog_config: dict[str, Any]) -> bool:
|
|
891
914
|
"""Updates configuration and data using given input catalog config
|
|
892
915
|
|
|
893
916
|
:param catalog_config: Catalog config, from yml stac_config[catalogs]
|
|
@@ -910,21 +933,36 @@ class StacCatalog(StacCommon):
|
|
|
910
933
|
|
|
911
934
|
return True
|
|
912
935
|
|
|
913
|
-
def
|
|
914
|
-
"""
|
|
936
|
+
def __build_stac_catalog(self, collection: Optional[str] = None) -> StacCatalog:
|
|
937
|
+
"""Build nested catalog from catalag list
|
|
915
938
|
|
|
916
|
-
:param
|
|
939
|
+
:param collection: (optional) product type id
|
|
940
|
+
:returns: This catalog obj
|
|
917
941
|
"""
|
|
918
|
-
|
|
919
|
-
|
|
920
|
-
|
|
921
|
-
|
|
922
|
-
|
|
923
|
-
|
|
942
|
+
settings = Settings.from_environment()
|
|
943
|
+
|
|
944
|
+
if not collection:
|
|
945
|
+
# Build root catalog combined with landing page
|
|
946
|
+
self.__update_data_from_catalog_config(
|
|
947
|
+
{
|
|
948
|
+
"model": {
|
|
949
|
+
**deepcopy(self.stac_config["landing_page"]),
|
|
950
|
+
**{
|
|
951
|
+
"provider": self.provider,
|
|
952
|
+
"id": settings.stac_api_landing_id,
|
|
953
|
+
"title": settings.stac_api_title,
|
|
954
|
+
"description": settings.stac_api_description,
|
|
955
|
+
},
|
|
956
|
+
}
|
|
957
|
+
}
|
|
958
|
+
)
|
|
959
|
+
else:
|
|
960
|
+
self.set_stac_product_type_by_id(collection)
|
|
961
|
+
return self
|
|
924
962
|
|
|
925
963
|
def set_stac_product_type_by_id(
|
|
926
964
|
self, product_type: str, **_: Any
|
|
927
|
-
) ->
|
|
965
|
+
) -> dict[str, Any]:
|
|
928
966
|
"""Updates catalog with given product_type
|
|
929
967
|
|
|
930
968
|
:param product_type: Product type
|
|
@@ -940,13 +978,23 @@ class StacCatalog(StacCommon):
|
|
|
940
978
|
if not collections:
|
|
941
979
|
raise NotAvailableError(f"Collection {product_type} does not exist.")
|
|
942
980
|
|
|
943
|
-
cat_model =
|
|
981
|
+
cat_model = {
|
|
982
|
+
"id": "{collection[id]}",
|
|
983
|
+
"title": "{collection[title]}",
|
|
984
|
+
"description": "{collection[description]}",
|
|
985
|
+
"extent": "{collection[extent]}",
|
|
986
|
+
"crs": "http://www.opengis.net/def/crs/OGC/1.3/CRS84",
|
|
987
|
+
"keywords": "{collection[keywords]}",
|
|
988
|
+
"license": "{collection[license]}",
|
|
989
|
+
"providers": "{collection[providers]}",
|
|
990
|
+
"summaries": "{collection[summaries]}",
|
|
991
|
+
}
|
|
944
992
|
# parse f-strings
|
|
945
993
|
format_args = deepcopy(self.stac_config)
|
|
946
994
|
format_args["catalog"] = defaultdict(str, **self.data)
|
|
947
995
|
format_args["collection"] = collections[0]
|
|
948
996
|
try:
|
|
949
|
-
parsed_dict:
|
|
997
|
+
parsed_dict: dict[str, Any] = format_dict_items(cat_model, **format_args)
|
|
950
998
|
except Exception:
|
|
951
999
|
logger.error("Could not format product_type catalog")
|
|
952
1000
|
raise
|
|
@@ -957,499 +1005,3 @@ class StacCatalog(StacCommon):
|
|
|
957
1005
|
self.search_args.update({"productType": product_type})
|
|
958
1006
|
|
|
959
1007
|
return parsed_dict
|
|
960
|
-
|
|
961
|
-
# get / set dates filters -------------------------------------------------
|
|
962
|
-
|
|
963
|
-
def get_stac_years_list(self, **_: Any) -> List[int]:
|
|
964
|
-
"""Get catalog available years list
|
|
965
|
-
|
|
966
|
-
:returns: Years list
|
|
967
|
-
"""
|
|
968
|
-
extent_date_min, extent_date_max = self.get_datetime_extent()
|
|
969
|
-
|
|
970
|
-
return list(range(extent_date_min.year, extent_date_max.year + 1))
|
|
971
|
-
|
|
972
|
-
def get_stac_months_list(self, **_: Any) -> List[int]:
|
|
973
|
-
"""Get catalog available months list
|
|
974
|
-
|
|
975
|
-
:returns: Months list
|
|
976
|
-
"""
|
|
977
|
-
extent_date_min, extent_date_max = self.get_datetime_extent()
|
|
978
|
-
|
|
979
|
-
return list(
|
|
980
|
-
range(
|
|
981
|
-
extent_date_min.month,
|
|
982
|
-
(extent_date_max - relativedelta(days=1)).month + 1,
|
|
983
|
-
)
|
|
984
|
-
)
|
|
985
|
-
|
|
986
|
-
def get_stac_days_list(self, **_: Any) -> List[int]:
|
|
987
|
-
"""Get catalog available days list
|
|
988
|
-
|
|
989
|
-
:returns: Days list
|
|
990
|
-
"""
|
|
991
|
-
extent_date_min, extent_date_max = self.get_datetime_extent()
|
|
992
|
-
|
|
993
|
-
return list(
|
|
994
|
-
range(
|
|
995
|
-
extent_date_min.day, (extent_date_max - relativedelta(days=1)).day + 1
|
|
996
|
-
)
|
|
997
|
-
)
|
|
998
|
-
|
|
999
|
-
def set_stac_year_by_id(self, year: str, **_: Any) -> Dict[str, Any]:
|
|
1000
|
-
"""Updates and returns catalog with given year
|
|
1001
|
-
|
|
1002
|
-
:param year: Year number
|
|
1003
|
-
:returns: Updated catalog
|
|
1004
|
-
"""
|
|
1005
|
-
extent_date_min, extent_date_max = self.get_datetime_extent()
|
|
1006
|
-
|
|
1007
|
-
datetime_min = max(
|
|
1008
|
-
[extent_date_min, dateutil.parser.parse(f"{year}-01-01T00:00:00Z")]
|
|
1009
|
-
)
|
|
1010
|
-
datetime_max = min(
|
|
1011
|
-
[
|
|
1012
|
-
extent_date_max,
|
|
1013
|
-
dateutil.parser.parse(f"{year}-01-01T00:00:00Z")
|
|
1014
|
-
+ relativedelta(years=1),
|
|
1015
|
-
]
|
|
1016
|
-
)
|
|
1017
|
-
|
|
1018
|
-
catalog_model = deepcopy(self.stac_config["catalogs"]["year"]["model"])
|
|
1019
|
-
|
|
1020
|
-
parsed_dict = self.set_stac_date(datetime_min, datetime_max, catalog_model)
|
|
1021
|
-
|
|
1022
|
-
return parsed_dict
|
|
1023
|
-
|
|
1024
|
-
def set_stac_month_by_id(self, month: str, **_: Any) -> Dict[str, Any]:
|
|
1025
|
-
"""Updates and returns catalog with given month
|
|
1026
|
-
|
|
1027
|
-
:param month: Month number
|
|
1028
|
-
:returns: Updated catalog
|
|
1029
|
-
"""
|
|
1030
|
-
extent_date_min, extent_date_max = self.get_datetime_extent()
|
|
1031
|
-
year = extent_date_min.year
|
|
1032
|
-
|
|
1033
|
-
datetime_min = max(
|
|
1034
|
-
[
|
|
1035
|
-
extent_date_min,
|
|
1036
|
-
dateutil.parser.parse(f"{year}-{month}-01T00:00:00Z"),
|
|
1037
|
-
]
|
|
1038
|
-
)
|
|
1039
|
-
datetime_max = min(
|
|
1040
|
-
[
|
|
1041
|
-
extent_date_max,
|
|
1042
|
-
dateutil.parser.parse(f"{year}-{month}-01T00:00:00Z")
|
|
1043
|
-
+ relativedelta(months=1),
|
|
1044
|
-
]
|
|
1045
|
-
)
|
|
1046
|
-
|
|
1047
|
-
catalog_model = deepcopy(self.stac_config["catalogs"]["month"]["model"])
|
|
1048
|
-
|
|
1049
|
-
parsed_dict = self.set_stac_date(datetime_min, datetime_max, catalog_model)
|
|
1050
|
-
|
|
1051
|
-
return parsed_dict
|
|
1052
|
-
|
|
1053
|
-
def set_stac_day_by_id(self, day: str, **_: Any) -> Dict[str, Any]:
|
|
1054
|
-
"""Updates and returns catalog with given day
|
|
1055
|
-
|
|
1056
|
-
:param day: Day number
|
|
1057
|
-
:returns: Updated catalog
|
|
1058
|
-
"""
|
|
1059
|
-
extent_date_min, extent_date_max = self.get_datetime_extent()
|
|
1060
|
-
year = extent_date_min.year
|
|
1061
|
-
month = extent_date_min.month
|
|
1062
|
-
|
|
1063
|
-
datetime_min = max(
|
|
1064
|
-
[
|
|
1065
|
-
extent_date_min,
|
|
1066
|
-
dateutil.parser.parse(f"{year}-{month}-{day}T00:00:00Z"),
|
|
1067
|
-
]
|
|
1068
|
-
)
|
|
1069
|
-
datetime_max = min(
|
|
1070
|
-
[
|
|
1071
|
-
extent_date_max,
|
|
1072
|
-
dateutil.parser.parse(f"{year}-{month}-{day}T00:00:00Z")
|
|
1073
|
-
+ relativedelta(days=1),
|
|
1074
|
-
]
|
|
1075
|
-
)
|
|
1076
|
-
|
|
1077
|
-
catalog_model = deepcopy(self.stac_config["catalogs"]["day"]["model"])
|
|
1078
|
-
|
|
1079
|
-
parsed_dict = self.set_stac_date(datetime_min, datetime_max, catalog_model)
|
|
1080
|
-
|
|
1081
|
-
return parsed_dict
|
|
1082
|
-
|
|
1083
|
-
def get_datetime_extent(self) -> Tuple[datetime, datetime]:
|
|
1084
|
-
"""Returns catalog temporal extent as datetime objs
|
|
1085
|
-
|
|
1086
|
-
:returns: Start & stop dates
|
|
1087
|
-
"""
|
|
1088
|
-
extent_date_min = dateutil.parser.parse(DEFAULT_MISSION_START_DATE).replace(
|
|
1089
|
-
tzinfo=tz.UTC
|
|
1090
|
-
)
|
|
1091
|
-
extent_date_max = datetime.now(timezone.utc).replace(tzinfo=tz.UTC)
|
|
1092
|
-
for interval in self.data["extent"]["temporal"]["interval"]:
|
|
1093
|
-
extent_date_min_str, extent_date_max_str = interval
|
|
1094
|
-
# date min
|
|
1095
|
-
if extent_date_min_str:
|
|
1096
|
-
extent_date_min = max(
|
|
1097
|
-
extent_date_min, dateutil.parser.parse(extent_date_min_str)
|
|
1098
|
-
)
|
|
1099
|
-
# date max
|
|
1100
|
-
if extent_date_max_str:
|
|
1101
|
-
extent_date_max = min(
|
|
1102
|
-
extent_date_max, dateutil.parser.parse(extent_date_max_str)
|
|
1103
|
-
)
|
|
1104
|
-
|
|
1105
|
-
return (
|
|
1106
|
-
extent_date_min.replace(tzinfo=tz.UTC),
|
|
1107
|
-
extent_date_max.replace(tzinfo=tz.UTC),
|
|
1108
|
-
)
|
|
1109
|
-
|
|
1110
|
-
def set_stac_date(
|
|
1111
|
-
self,
|
|
1112
|
-
datetime_min: datetime,
|
|
1113
|
-
datetime_max: datetime,
|
|
1114
|
-
catalog_model: Dict[str, Any],
|
|
1115
|
-
):
|
|
1116
|
-
"""Updates catalog data using given dates
|
|
1117
|
-
|
|
1118
|
-
:param datetime_min: Date min of interval
|
|
1119
|
-
:param datetime_max: Date max of interval
|
|
1120
|
-
:param catalog_model: Catalog model to use, from yml stac_config[catalogs]
|
|
1121
|
-
:returns: Updated catalog
|
|
1122
|
-
"""
|
|
1123
|
-
# parse f-strings
|
|
1124
|
-
format_args = deepcopy(self.stac_config)
|
|
1125
|
-
format_args["catalog"] = defaultdict(str, **self.data)
|
|
1126
|
-
format_args["date"] = defaultdict(
|
|
1127
|
-
str,
|
|
1128
|
-
{
|
|
1129
|
-
"year": datetime_min.year,
|
|
1130
|
-
"month": datetime_min.month,
|
|
1131
|
-
"day": datetime_min.day,
|
|
1132
|
-
"min": datetime_min.isoformat().replace("+00:00", "Z"),
|
|
1133
|
-
"max": datetime_max.isoformat().replace("+00:00", "Z"),
|
|
1134
|
-
},
|
|
1135
|
-
)
|
|
1136
|
-
parsed_dict: Dict[str, Any] = format_dict_items(catalog_model, **format_args)
|
|
1137
|
-
|
|
1138
|
-
self.update_data(parsed_dict)
|
|
1139
|
-
|
|
1140
|
-
# update search args
|
|
1141
|
-
self.search_args.update(
|
|
1142
|
-
{
|
|
1143
|
-
"start": datetime_min.isoformat().replace("+00:00", "Z"),
|
|
1144
|
-
"end": datetime_max.isoformat().replace("+00:00", "Z"),
|
|
1145
|
-
}
|
|
1146
|
-
)
|
|
1147
|
-
return parsed_dict
|
|
1148
|
-
|
|
1149
|
-
# get / set cloud_cover filter --------------------------------------------
|
|
1150
|
-
|
|
1151
|
-
def get_stac_cloud_covers_list(self, **_: Any) -> List[int]:
|
|
1152
|
-
"""Get cloud_cover list
|
|
1153
|
-
|
|
1154
|
-
:returns: cloud_cover list
|
|
1155
|
-
"""
|
|
1156
|
-
return list(range(0, 101, 10))
|
|
1157
|
-
|
|
1158
|
-
def set_stac_cloud_cover_by_id(self, cloud_cover: str, **_: Any) -> Dict[str, Any]:
|
|
1159
|
-
"""Updates and returns catalog with given max cloud_cover
|
|
1160
|
-
|
|
1161
|
-
:param cloud_cover: Cloud_cover number
|
|
1162
|
-
:returns: Updated catalog
|
|
1163
|
-
"""
|
|
1164
|
-
cat_model = deepcopy(self.stac_config["catalogs"]["cloud_cover"]["model"])
|
|
1165
|
-
# parse f-strings
|
|
1166
|
-
format_args = deepcopy(self.stac_config)
|
|
1167
|
-
format_args["catalog"] = defaultdict(str, **self.data)
|
|
1168
|
-
format_args["cloud_cover"] = cloud_cover
|
|
1169
|
-
parsed_dict: Dict[str, Any] = format_dict_items(cat_model, **format_args)
|
|
1170
|
-
|
|
1171
|
-
self.update_data(parsed_dict)
|
|
1172
|
-
|
|
1173
|
-
# update search args
|
|
1174
|
-
self.search_args.update({"cloudCover": cloud_cover})
|
|
1175
|
-
|
|
1176
|
-
return parsed_dict
|
|
1177
|
-
|
|
1178
|
-
# get / set locations filter ----------------------------------------------
|
|
1179
|
-
|
|
1180
|
-
def get_stac_location_list(self, catalog_name: str) -> List[str]:
|
|
1181
|
-
"""Get locations list using stac_conf & locations_config
|
|
1182
|
-
|
|
1183
|
-
:param catalog_name: Catalog/location name
|
|
1184
|
-
:returns: Locations list
|
|
1185
|
-
"""
|
|
1186
|
-
|
|
1187
|
-
if catalog_name not in self.stac_config["catalogs"]:
|
|
1188
|
-
logger.warning("no entry found for %s in location_config", catalog_name)
|
|
1189
|
-
return []
|
|
1190
|
-
location_config = self.stac_config["catalogs"][catalog_name]
|
|
1191
|
-
|
|
1192
|
-
for k in ["path", "attr"]:
|
|
1193
|
-
if k not in location_config.keys():
|
|
1194
|
-
logger.warning(
|
|
1195
|
-
"no %s key found for %s in location_config", k, catalog_name
|
|
1196
|
-
)
|
|
1197
|
-
return []
|
|
1198
|
-
path = location_config["path"]
|
|
1199
|
-
attr = location_config["attr"]
|
|
1200
|
-
|
|
1201
|
-
with shapefile.Reader(path) as shp:
|
|
1202
|
-
countries_list: List[str] = [rec[attr] for rec in shp.records()] # type: ignore
|
|
1203
|
-
|
|
1204
|
-
# remove duplicates
|
|
1205
|
-
countries_list = list(set(countries_list))
|
|
1206
|
-
|
|
1207
|
-
countries_list.sort()
|
|
1208
|
-
|
|
1209
|
-
return countries_list
|
|
1210
|
-
|
|
1211
|
-
def set_stac_location_by_id(
|
|
1212
|
-
self, location: str, catalog_name: str
|
|
1213
|
-
) -> Dict[str, Any]:
|
|
1214
|
-
"""Updates and returns catalog with given location
|
|
1215
|
-
|
|
1216
|
-
:param location: Feature attribute value for shp filtering
|
|
1217
|
-
:param catalog_name: Catalog/location name
|
|
1218
|
-
:returns: Updated catalog
|
|
1219
|
-
"""
|
|
1220
|
-
location_list_cat_key = catalog_name + "_list"
|
|
1221
|
-
|
|
1222
|
-
if location_list_cat_key not in self.stac_config["catalogs"]:
|
|
1223
|
-
logger.warning(
|
|
1224
|
-
"no entry found for %s's list in location_config", catalog_name
|
|
1225
|
-
)
|
|
1226
|
-
return {}
|
|
1227
|
-
location_config = self.stac_config["catalogs"][location_list_cat_key]
|
|
1228
|
-
|
|
1229
|
-
for k in ["path", "attr"]:
|
|
1230
|
-
if k not in location_config.keys():
|
|
1231
|
-
logger.warning(
|
|
1232
|
-
"no %s key found for %s's list in location_config", k, catalog_name
|
|
1233
|
-
)
|
|
1234
|
-
return {}
|
|
1235
|
-
path = location_config["path"]
|
|
1236
|
-
attr = location_config["attr"]
|
|
1237
|
-
|
|
1238
|
-
with shapefile.Reader(path) as shp:
|
|
1239
|
-
geom_hits = [
|
|
1240
|
-
shape(shaperec.shape)
|
|
1241
|
-
for shaperec in shp.shapeRecords()
|
|
1242
|
-
if shaperec.record.as_dict().get(attr, None) == location
|
|
1243
|
-
]
|
|
1244
|
-
|
|
1245
|
-
if not geom_hits:
|
|
1246
|
-
logger.warning(
|
|
1247
|
-
"no feature found in %s matching %s=%s", path, attr, location
|
|
1248
|
-
)
|
|
1249
|
-
return {}
|
|
1250
|
-
|
|
1251
|
-
geom = cast(BaseGeometry, unary_union(geom_hits))
|
|
1252
|
-
|
|
1253
|
-
cat_model = deepcopy(self.stac_config["catalogs"]["country"]["model"])
|
|
1254
|
-
# parse f-strings
|
|
1255
|
-
format_args = deepcopy(self.stac_config)
|
|
1256
|
-
format_args["catalog"] = defaultdict(str, **self.data)
|
|
1257
|
-
format_args["feature"] = defaultdict(str, {"geometry": geom, "id": location})
|
|
1258
|
-
parsed_dict: Dict[str, Any] = format_dict_items(cat_model, **format_args)
|
|
1259
|
-
|
|
1260
|
-
self.update_data(parsed_dict)
|
|
1261
|
-
|
|
1262
|
-
# update search args
|
|
1263
|
-
self.search_args.update({"geom": geom})
|
|
1264
|
-
|
|
1265
|
-
return parsed_dict
|
|
1266
|
-
|
|
1267
|
-
def build_locations_config(self) -> Dict[str, str]:
|
|
1268
|
-
"""Build locations config from stac_conf[locations_catalogs] & eodag_api.locations_config
|
|
1269
|
-
|
|
1270
|
-
:returns: Locations configuration dict
|
|
1271
|
-
"""
|
|
1272
|
-
user_config_locations_list = self.eodag_api.locations_config
|
|
1273
|
-
|
|
1274
|
-
locations_config_model = deepcopy(self.stac_config["locations_catalogs"])
|
|
1275
|
-
|
|
1276
|
-
locations_config: Dict[str, str] = {}
|
|
1277
|
-
for loc in user_config_locations_list:
|
|
1278
|
-
# parse jsonpath
|
|
1279
|
-
parsed = jsonpath_parse_dict_items(
|
|
1280
|
-
locations_config_model, {"shp_location": loc}
|
|
1281
|
-
)
|
|
1282
|
-
|
|
1283
|
-
# set default child/parent for this location
|
|
1284
|
-
parsed["location"]["parent_key"] = f"{loc['name']}_list"
|
|
1285
|
-
|
|
1286
|
-
locations_config[f"{loc['name']}_list"] = parsed["locations_list"]
|
|
1287
|
-
locations_config[loc["name"]] = parsed["location"]
|
|
1288
|
-
|
|
1289
|
-
return locations_config
|
|
1290
|
-
|
|
1291
|
-
def __build_stac_catalog(self, catalogs: Optional[List[str]] = None) -> StacCatalog:
|
|
1292
|
-
"""Build nested catalog from catalag list
|
|
1293
|
-
|
|
1294
|
-
:param catalogs: (optional) Catalogs list
|
|
1295
|
-
:returns: This catalog obj
|
|
1296
|
-
"""
|
|
1297
|
-
settings = Settings.from_environment()
|
|
1298
|
-
|
|
1299
|
-
# update conf with user shp locations
|
|
1300
|
-
locations_config = self.build_locations_config()
|
|
1301
|
-
|
|
1302
|
-
self.stac_config["catalogs"] = {
|
|
1303
|
-
**deepcopy(self.stac_config["catalogs"]),
|
|
1304
|
-
**locations_config,
|
|
1305
|
-
}
|
|
1306
|
-
|
|
1307
|
-
if not catalogs:
|
|
1308
|
-
# Build root catalog combined with landing page
|
|
1309
|
-
self.__update_data_from_catalog_config(
|
|
1310
|
-
{
|
|
1311
|
-
"model": {
|
|
1312
|
-
**deepcopy(self.stac_config["landing_page"]),
|
|
1313
|
-
**{
|
|
1314
|
-
"provider": self.provider,
|
|
1315
|
-
"id": settings.stac_api_landing_id,
|
|
1316
|
-
"title": settings.stac_api_title,
|
|
1317
|
-
"description": settings.stac_api_description,
|
|
1318
|
-
},
|
|
1319
|
-
}
|
|
1320
|
-
}
|
|
1321
|
-
)
|
|
1322
|
-
|
|
1323
|
-
# build children : product_types
|
|
1324
|
-
product_types_list = [
|
|
1325
|
-
pt
|
|
1326
|
-
for pt in self.eodag_api.list_product_types(
|
|
1327
|
-
provider=self.provider, fetch_providers=False
|
|
1328
|
-
)
|
|
1329
|
-
]
|
|
1330
|
-
self.set_children(
|
|
1331
|
-
[
|
|
1332
|
-
{
|
|
1333
|
-
"rel": "child",
|
|
1334
|
-
"href": urljoin(
|
|
1335
|
-
self.url, f"{STAC_CATALOGS_PREFIX}/{product_type['ID']}"
|
|
1336
|
-
),
|
|
1337
|
-
"title": product_type["title"],
|
|
1338
|
-
}
|
|
1339
|
-
for product_type in product_types_list
|
|
1340
|
-
]
|
|
1341
|
-
)
|
|
1342
|
-
return self
|
|
1343
|
-
|
|
1344
|
-
# use product_types_list as base for building nested catalogs
|
|
1345
|
-
self.__update_data_from_catalog_config(
|
|
1346
|
-
deepcopy(self.stac_config["catalogs"]["product_types_list"])
|
|
1347
|
-
)
|
|
1348
|
-
|
|
1349
|
-
for idx, cat in enumerate(catalogs):
|
|
1350
|
-
if idx % 2 == 0:
|
|
1351
|
-
# even: cat is a filtering value ----------------------------------
|
|
1352
|
-
cat_data_name = self.catalog_config["child_key"]
|
|
1353
|
-
cat_data_value = cat
|
|
1354
|
-
|
|
1355
|
-
# update data
|
|
1356
|
-
cat_data_name_dict = self.stac_config["catalogs"][cat_data_name]
|
|
1357
|
-
set_data_method_name = (
|
|
1358
|
-
f"set_stac_{cat_data_name}_by_id"
|
|
1359
|
-
if "catalog_type" not in cat_data_name_dict.keys()
|
|
1360
|
-
else f"set_stac_{cat_data_name_dict['catalog_type']}_by_id"
|
|
1361
|
-
)
|
|
1362
|
-
set_data_method = getattr(self, set_data_method_name)
|
|
1363
|
-
set_data_method(cat_data_value, catalog_name=cat_data_name)
|
|
1364
|
-
|
|
1365
|
-
if idx == len(catalogs) - 1:
|
|
1366
|
-
# build children : remaining filtering keys
|
|
1367
|
-
remaining_catalogs_list = [
|
|
1368
|
-
c
|
|
1369
|
-
for c in self.stac_config["catalogs"].keys()
|
|
1370
|
-
# keep filters not used yet AND
|
|
1371
|
-
if self.stac_config["catalogs"][c]["model"]["id"]
|
|
1372
|
-
not in catalogs
|
|
1373
|
-
and (
|
|
1374
|
-
# filters with no parent_key constraint (no key, or key=None) OR
|
|
1375
|
-
"parent_key" not in self.stac_config["catalogs"][c]
|
|
1376
|
-
or not self.stac_config["catalogs"][c]["parent_key"]
|
|
1377
|
-
# filters matching parent_key constraint
|
|
1378
|
-
or self.stac_config["catalogs"][c]["parent_key"]
|
|
1379
|
-
== cat_data_name
|
|
1380
|
-
)
|
|
1381
|
-
# AND filters that match parent attr constraint (locations)
|
|
1382
|
-
and (
|
|
1383
|
-
"parent" not in self.stac_config["catalogs"][c]
|
|
1384
|
-
or not self.stac_config["catalogs"][c]["parent"]["key"]
|
|
1385
|
-
or (
|
|
1386
|
-
self.stac_config["catalogs"][c]["parent"]["key"]
|
|
1387
|
-
== cat_data_name
|
|
1388
|
-
and self.stac_config["catalogs"][c]["parent"]["attr"]
|
|
1389
|
-
== cat_data_value
|
|
1390
|
-
)
|
|
1391
|
-
)
|
|
1392
|
-
]
|
|
1393
|
-
|
|
1394
|
-
self.set_children(
|
|
1395
|
-
[
|
|
1396
|
-
{
|
|
1397
|
-
"rel": "child",
|
|
1398
|
-
"href": self.url
|
|
1399
|
-
+ "/"
|
|
1400
|
-
+ self.stac_config["catalogs"][c]["model"]["id"],
|
|
1401
|
-
"title": str(
|
|
1402
|
-
self.stac_config["catalogs"][c]["model"]["id"]
|
|
1403
|
-
),
|
|
1404
|
-
}
|
|
1405
|
-
for c in remaining_catalogs_list
|
|
1406
|
-
]
|
|
1407
|
-
+ [
|
|
1408
|
-
{
|
|
1409
|
-
"rel": "items",
|
|
1410
|
-
"href": self.url + "/items",
|
|
1411
|
-
"title": "items",
|
|
1412
|
-
}
|
|
1413
|
-
]
|
|
1414
|
-
)
|
|
1415
|
-
|
|
1416
|
-
else:
|
|
1417
|
-
# odd: cat is a filtering key -------------------------------------
|
|
1418
|
-
try:
|
|
1419
|
-
cat_key = [
|
|
1420
|
-
c
|
|
1421
|
-
for c in self.stac_config["catalogs"].keys()
|
|
1422
|
-
if self.stac_config["catalogs"][c]["model"]["id"] == cat
|
|
1423
|
-
][0]
|
|
1424
|
-
except IndexError as e:
|
|
1425
|
-
raise ValidationError(
|
|
1426
|
-
f"Bad settings for {cat} in stac_config catalogs"
|
|
1427
|
-
) from e
|
|
1428
|
-
cat_config = deepcopy(self.stac_config["catalogs"][cat_key])
|
|
1429
|
-
# update data
|
|
1430
|
-
self.__update_data_from_catalog_config(cat_config)
|
|
1431
|
-
|
|
1432
|
-
# get filtering values list
|
|
1433
|
-
get_data_method_name = (
|
|
1434
|
-
f"get_stac_{cat_key}"
|
|
1435
|
-
if "catalog_type"
|
|
1436
|
-
not in self.stac_config["catalogs"][cat_key].keys()
|
|
1437
|
-
else f"get_stac_{self.stac_config['catalogs'][cat_key]['catalog_type']}"
|
|
1438
|
-
)
|
|
1439
|
-
get_data_method = getattr(self, get_data_method_name)
|
|
1440
|
-
cat_data_list = get_data_method(catalog_name=cat_key)
|
|
1441
|
-
|
|
1442
|
-
if idx == len(catalogs) - 1:
|
|
1443
|
-
# filtering values list as children (do not include items)
|
|
1444
|
-
self.set_children(
|
|
1445
|
-
[
|
|
1446
|
-
{
|
|
1447
|
-
"rel": "child",
|
|
1448
|
-
"href": self.url + "/" + str(filtering_data),
|
|
1449
|
-
"title": str(filtering_data),
|
|
1450
|
-
}
|
|
1451
|
-
for filtering_data in cat_data_list
|
|
1452
|
-
]
|
|
1453
|
-
)
|
|
1454
|
-
|
|
1455
|
-
return self
|