eodag 2.12.1__py3-none-any.whl → 3.0.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- eodag/__init__.py +6 -8
- eodag/api/core.py +654 -538
- eodag/api/product/__init__.py +12 -2
- eodag/api/product/_assets.py +59 -16
- eodag/api/product/_product.py +100 -93
- eodag/api/product/drivers/__init__.py +7 -2
- eodag/api/product/drivers/base.py +0 -3
- eodag/api/product/metadata_mapping.py +192 -96
- eodag/api/search_result.py +69 -10
- eodag/cli.py +55 -25
- eodag/config.py +391 -116
- eodag/plugins/apis/base.py +11 -168
- eodag/plugins/apis/ecmwf.py +36 -25
- eodag/plugins/apis/usgs.py +80 -35
- eodag/plugins/authentication/aws_auth.py +13 -4
- eodag/plugins/authentication/base.py +10 -1
- eodag/plugins/authentication/generic.py +2 -2
- eodag/plugins/authentication/header.py +31 -6
- eodag/plugins/authentication/keycloak.py +17 -84
- eodag/plugins/authentication/oauth.py +3 -3
- eodag/plugins/authentication/openid_connect.py +268 -49
- eodag/plugins/authentication/qsauth.py +4 -1
- eodag/plugins/authentication/sas_auth.py +9 -2
- eodag/plugins/authentication/token.py +98 -47
- eodag/plugins/authentication/token_exchange.py +122 -0
- eodag/plugins/crunch/base.py +3 -1
- eodag/plugins/crunch/filter_date.py +3 -9
- eodag/plugins/crunch/filter_latest_intersect.py +0 -3
- eodag/plugins/crunch/filter_latest_tpl_name.py +1 -4
- eodag/plugins/crunch/filter_overlap.py +4 -8
- eodag/plugins/crunch/filter_property.py +5 -11
- eodag/plugins/download/aws.py +149 -185
- eodag/plugins/download/base.py +88 -97
- eodag/plugins/download/creodias_s3.py +1 -1
- eodag/plugins/download/http.py +638 -310
- eodag/plugins/download/s3rest.py +47 -45
- eodag/plugins/manager.py +228 -88
- eodag/plugins/search/__init__.py +36 -0
- eodag/plugins/search/base.py +239 -30
- eodag/plugins/search/build_search_result.py +382 -37
- eodag/plugins/search/cop_marine.py +441 -0
- eodag/plugins/search/creodias_s3.py +25 -20
- eodag/plugins/search/csw.py +5 -7
- eodag/plugins/search/data_request_search.py +61 -30
- eodag/plugins/search/qssearch.py +713 -255
- eodag/plugins/search/static_stac_search.py +106 -40
- eodag/resources/ext_product_types.json +1 -1
- eodag/resources/product_types.yml +1921 -34
- eodag/resources/providers.yml +4091 -3655
- eodag/resources/stac.yml +50 -216
- eodag/resources/stac_api.yml +71 -25
- eodag/resources/stac_provider.yml +5 -0
- eodag/resources/user_conf_template.yml +89 -32
- eodag/rest/__init__.py +6 -0
- eodag/rest/cache.py +70 -0
- eodag/rest/config.py +68 -0
- eodag/rest/constants.py +26 -0
- eodag/rest/core.py +735 -0
- eodag/rest/errors.py +178 -0
- eodag/rest/server.py +264 -431
- eodag/rest/stac.py +442 -836
- eodag/rest/types/collections_search.py +44 -0
- eodag/rest/types/eodag_search.py +238 -47
- eodag/rest/types/queryables.py +164 -0
- eodag/rest/types/stac_search.py +273 -0
- eodag/rest/utils/__init__.py +216 -0
- eodag/rest/utils/cql_evaluate.py +119 -0
- eodag/rest/utils/rfc3339.py +64 -0
- eodag/types/__init__.py +106 -10
- eodag/types/bbox.py +15 -14
- eodag/types/download_args.py +40 -0
- eodag/types/search_args.py +57 -7
- eodag/types/whoosh.py +79 -0
- eodag/utils/__init__.py +110 -91
- eodag/utils/constraints.py +37 -45
- eodag/utils/exceptions.py +39 -22
- eodag/utils/import_system.py +0 -4
- eodag/utils/logging.py +37 -80
- eodag/utils/notebook.py +4 -4
- eodag/utils/repr.py +113 -0
- eodag/utils/requests.py +128 -0
- eodag/utils/rest.py +100 -0
- eodag/utils/stac_reader.py +93 -21
- {eodag-2.12.1.dist-info → eodag-3.0.0.dist-info}/METADATA +88 -53
- eodag-3.0.0.dist-info/RECORD +109 -0
- {eodag-2.12.1.dist-info → eodag-3.0.0.dist-info}/WHEEL +1 -1
- {eodag-2.12.1.dist-info → eodag-3.0.0.dist-info}/entry_points.txt +7 -5
- eodag/plugins/apis/cds.py +0 -540
- eodag/rest/types/stac_queryables.py +0 -134
- eodag/rest/utils.py +0 -1133
- eodag-2.12.1.dist-info/RECORD +0 -94
- {eodag-2.12.1.dist-info → eodag-3.0.0.dist-info}/LICENSE +0 -0
- {eodag-2.12.1.dist-info → eodag-3.0.0.dist-info}/top_level.txt +0 -0
eodag/rest/stac.py
CHANGED
|
@@ -21,37 +21,43 @@ import logging
|
|
|
21
21
|
import os
|
|
22
22
|
from collections import defaultdict
|
|
23
23
|
from datetime import datetime, timezone
|
|
24
|
-
from typing import TYPE_CHECKING, Any, Dict, List, Optional
|
|
25
|
-
from urllib.parse import
|
|
24
|
+
from typing import TYPE_CHECKING, Any, Dict, List, Optional
|
|
25
|
+
from urllib.parse import (
|
|
26
|
+
parse_qs,
|
|
27
|
+
quote,
|
|
28
|
+
urlencode,
|
|
29
|
+
urlparse,
|
|
30
|
+
urlsplit,
|
|
31
|
+
urlunparse,
|
|
32
|
+
urlunsplit,
|
|
33
|
+
)
|
|
26
34
|
|
|
27
|
-
import dateutil.parser
|
|
28
35
|
import geojson
|
|
29
|
-
import
|
|
30
|
-
from dateutil import tz
|
|
31
|
-
from dateutil.relativedelta import relativedelta
|
|
32
|
-
from shapely.geometry import shape
|
|
33
|
-
from shapely.geometry.base import BaseGeometry
|
|
34
|
-
from shapely.ops import unary_union
|
|
36
|
+
from jsonpath_ng.jsonpath import Child
|
|
35
37
|
|
|
36
38
|
from eodag.api.product.metadata_mapping import (
|
|
37
39
|
DEFAULT_METADATA_MAPPING,
|
|
38
40
|
format_metadata,
|
|
39
41
|
get_metadata_path,
|
|
40
42
|
)
|
|
43
|
+
from eodag.rest.config import Settings
|
|
44
|
+
from eodag.rest.utils.rfc3339 import str_to_interval
|
|
41
45
|
from eodag.utils import (
|
|
42
46
|
deepcopy,
|
|
43
47
|
dict_items_recursive_apply,
|
|
44
48
|
format_dict_items,
|
|
49
|
+
guess_file_type,
|
|
45
50
|
jsonpath_parse_dict_items,
|
|
46
51
|
string_to_jsonpath,
|
|
47
52
|
update_nested_dict,
|
|
48
|
-
urljoin,
|
|
49
53
|
)
|
|
50
54
|
from eodag.utils.exceptions import (
|
|
51
55
|
NoMatchingProductType,
|
|
52
56
|
NotAvailableError,
|
|
53
|
-
|
|
57
|
+
RequestError,
|
|
58
|
+
TimeOutError,
|
|
54
59
|
)
|
|
60
|
+
from eodag.utils.requests import fetch_json
|
|
55
61
|
|
|
56
62
|
if TYPE_CHECKING:
|
|
57
63
|
from eodag.api.core import EODataAccessGateway
|
|
@@ -61,23 +67,52 @@ if TYPE_CHECKING:
|
|
|
61
67
|
|
|
62
68
|
logger = logging.getLogger("eodag.rest.stac")
|
|
63
69
|
|
|
64
|
-
|
|
65
|
-
|
|
70
|
+
# fields not to put in item properties
|
|
71
|
+
COLLECTION_PROPERTIES = [
|
|
72
|
+
"abstract",
|
|
73
|
+
"instrument",
|
|
74
|
+
"platform",
|
|
75
|
+
"platformSerialIdentifier",
|
|
76
|
+
"processingLevel",
|
|
77
|
+
"sensorType",
|
|
78
|
+
"md5",
|
|
79
|
+
"license",
|
|
80
|
+
"title",
|
|
81
|
+
"missionStartDate",
|
|
82
|
+
"missionEndDate",
|
|
83
|
+
"keywords",
|
|
84
|
+
"stacCollection",
|
|
85
|
+
]
|
|
86
|
+
IGNORED_ITEM_PROPERTIES = [
|
|
87
|
+
"_id",
|
|
88
|
+
"id",
|
|
89
|
+
"keyword",
|
|
90
|
+
"quicklook",
|
|
91
|
+
"thumbnail",
|
|
92
|
+
"downloadLink",
|
|
93
|
+
"orderLink",
|
|
94
|
+
"_dc_qs",
|
|
95
|
+
"qs",
|
|
96
|
+
"defaultGeometry",
|
|
97
|
+
"_date",
|
|
98
|
+
]
|
|
99
|
+
|
|
100
|
+
|
|
101
|
+
def _quote_url_path(url: str) -> str:
|
|
102
|
+
parsed = urlsplit(url)
|
|
103
|
+
path = quote(parsed.path)
|
|
104
|
+
components = (parsed.scheme, parsed.netloc, path, parsed.query, parsed.fragment)
|
|
105
|
+
return urlunsplit(components)
|
|
66
106
|
|
|
67
107
|
|
|
68
108
|
class StacCommon:
|
|
69
109
|
"""Stac common object
|
|
70
110
|
|
|
71
111
|
:param url: Requested URL
|
|
72
|
-
:type url: str
|
|
73
112
|
:param stac_config: STAC configuration from stac.yml conf file
|
|
74
|
-
:type stac_config: dict
|
|
75
113
|
:param provider: (optional) Chosen provider
|
|
76
|
-
:type provider: str
|
|
77
114
|
:param eodag_api: EODAG python API instance
|
|
78
|
-
:type eodag_api: :class:`eodag.api.core.EODataAccessGateway`
|
|
79
115
|
:param root: (optional) API root
|
|
80
|
-
:type root: str
|
|
81
116
|
"""
|
|
82
117
|
|
|
83
118
|
def __init__(
|
|
@@ -100,7 +135,6 @@ class StacCommon:
|
|
|
100
135
|
"""Updates data using given input STAC dict data
|
|
101
136
|
|
|
102
137
|
:param data: Catalog data (parsed STAC dict)
|
|
103
|
-
:type data: dict
|
|
104
138
|
"""
|
|
105
139
|
self.data.update(data)
|
|
106
140
|
|
|
@@ -112,15 +146,18 @@ class StacCommon:
|
|
|
112
146
|
):
|
|
113
147
|
for i, bbox in enumerate(self.data["extent"]["spatial"]["bbox"]):
|
|
114
148
|
self.data["extent"]["spatial"]["bbox"][i] = [float(x) for x in bbox]
|
|
115
|
-
# "None" values to None
|
|
116
|
-
self.data = dict_items_recursive_apply(
|
|
117
|
-
self.data, lambda k, v: None if v == "None" else v
|
|
118
|
-
)
|
|
119
149
|
|
|
120
|
-
|
|
121
|
-
|
|
122
|
-
|
|
123
|
-
|
|
150
|
+
def apply_method_none(_: str, v: str) -> Optional[str]:
|
|
151
|
+
""" "None" values to None"""
|
|
152
|
+
return None if v == "None" else v
|
|
153
|
+
|
|
154
|
+
self.data = dict_items_recursive_apply(self.data, apply_method_none)
|
|
155
|
+
|
|
156
|
+
def apply_method_ids(k, v):
|
|
157
|
+
"""ids and titles as str"""
|
|
158
|
+
return str(v) if k in ["title", "id"] else v
|
|
159
|
+
|
|
160
|
+
self.data = dict_items_recursive_apply(self.data, apply_method_ids)
|
|
124
161
|
|
|
125
162
|
# empty stac_extensions: "" to []
|
|
126
163
|
if not self.data.get("stac_extensions", True):
|
|
@@ -133,15 +170,10 @@ class StacCommon:
|
|
|
133
170
|
"""Parse STAC extension from config and return as dict
|
|
134
171
|
|
|
135
172
|
:param url: Requested URL
|
|
136
|
-
:type url: str
|
|
137
173
|
:param stac_config: STAC configuration from stac.yml conf file
|
|
138
|
-
:type stac_config: dict
|
|
139
174
|
:param extension: Extension name
|
|
140
|
-
:type extension: str
|
|
141
175
|
:param kwargs: Additional variables needed for parsing extension
|
|
142
|
-
:
|
|
143
|
-
:returns: STAC extension as dictionnary
|
|
144
|
-
:rtype: dict
|
|
176
|
+
:returns: STAC extension as dictionary
|
|
145
177
|
"""
|
|
146
178
|
extension_model = deepcopy(stac_config).get("extensions", {}).get(extension, {})
|
|
147
179
|
|
|
@@ -153,30 +185,30 @@ class StacCommon:
|
|
|
153
185
|
}
|
|
154
186
|
return format_dict_items(extension_model, **format_args)
|
|
155
187
|
|
|
156
|
-
def
|
|
157
|
-
"""
|
|
158
|
-
|
|
159
|
-
|
|
160
|
-
|
|
161
|
-
|
|
162
|
-
|
|
163
|
-
|
|
164
|
-
|
|
188
|
+
def get_provider_dict(self, provider: str) -> Dict[str, Any]:
|
|
189
|
+
"""Generate STAC provider dict"""
|
|
190
|
+
provider_config = next(
|
|
191
|
+
p
|
|
192
|
+
for p in self.eodag_api.providers_config.values()
|
|
193
|
+
if provider in [p.name, getattr(p, "group", None)]
|
|
194
|
+
)
|
|
195
|
+
return {
|
|
196
|
+
"name": getattr(provider_config, "group", provider_config.name),
|
|
197
|
+
"description": getattr(provider_config, "description", None),
|
|
198
|
+
"roles": getattr(provider_config, "roles", None),
|
|
199
|
+
"url": getattr(provider_config, "url", None),
|
|
200
|
+
"priority": getattr(provider_config, "priority", None),
|
|
201
|
+
}
|
|
165
202
|
|
|
166
203
|
|
|
167
204
|
class StacItem(StacCommon):
|
|
168
205
|
"""Stac item object
|
|
169
206
|
|
|
170
207
|
:param url: Requested URL
|
|
171
|
-
:type url: str
|
|
172
208
|
:param stac_config: STAC configuration from stac.yml conf file
|
|
173
|
-
:type stac_config: dict
|
|
174
209
|
:param provider: (optional) Chosen provider
|
|
175
|
-
:type provider: str
|
|
176
210
|
:param eodag_api: EODAG python API instance
|
|
177
|
-
:type eodag_api: :class:`eodag.api.core.EODataAccessGateway`
|
|
178
211
|
:param root: (optional) API root
|
|
179
|
-
:type root: str
|
|
180
212
|
"""
|
|
181
213
|
|
|
182
214
|
def __init__(
|
|
@@ -201,11 +233,8 @@ class StacItem(StacCommon):
|
|
|
201
233
|
"""Build STAC items list from EODAG search results
|
|
202
234
|
|
|
203
235
|
:param search_results: EODAG search results
|
|
204
|
-
:type search_results: :class:`~eodag.api.search_result.SearchResult`
|
|
205
236
|
:param catalog: STAC catalog dict used for parsing item metadata
|
|
206
|
-
:type catalog: dict
|
|
207
237
|
:returns: STAC item dicts list
|
|
208
|
-
:rtype: list
|
|
209
238
|
"""
|
|
210
239
|
if len(search_results) <= 0:
|
|
211
240
|
return []
|
|
@@ -213,7 +242,6 @@ class StacItem(StacCommon):
|
|
|
213
242
|
item_model = self.__filter_item_model_properties(
|
|
214
243
|
self.stac_config["item"], str(search_results[0].product_type)
|
|
215
244
|
)
|
|
216
|
-
provider_model = deepcopy(self.stac_config["provider"])
|
|
217
245
|
|
|
218
246
|
# check if some items need to be converted
|
|
219
247
|
need_conversion: Dict[str, Any] = {}
|
|
@@ -229,77 +257,62 @@ class StacItem(StacCommon):
|
|
|
229
257
|
k, item_model["properties"][k]
|
|
230
258
|
)
|
|
231
259
|
|
|
260
|
+
item_props = [
|
|
261
|
+
p.right.fields[0]
|
|
262
|
+
for p in item_model["properties"].values()
|
|
263
|
+
if isinstance(p, Child)
|
|
264
|
+
]
|
|
265
|
+
ignored_props = COLLECTION_PROPERTIES + item_props + IGNORED_ITEM_PROPERTIES
|
|
266
|
+
|
|
232
267
|
item_list: List[Dict[str, Any]] = []
|
|
233
268
|
for product in search_results:
|
|
234
|
-
# parse jsonpath
|
|
235
|
-
provider_dict = jsonpath_parse_dict_items(
|
|
236
|
-
provider_model,
|
|
237
|
-
{
|
|
238
|
-
"provider": self.eodag_api.providers_config[
|
|
239
|
-
product.provider
|
|
240
|
-
].__dict__
|
|
241
|
-
},
|
|
242
|
-
)
|
|
243
|
-
|
|
244
269
|
product_dict = deepcopy(product.__dict__)
|
|
245
|
-
if isinstance(product.assets, dict):
|
|
246
|
-
product_dict["assets"] = product.assets
|
|
247
|
-
else:
|
|
248
|
-
product_dict["assets"] = product.assets.as_dict()
|
|
249
270
|
|
|
250
|
-
product_item = jsonpath_parse_dict_items(
|
|
271
|
+
product_item: Dict[str, Any] = jsonpath_parse_dict_items(
|
|
251
272
|
item_model,
|
|
252
273
|
{
|
|
253
274
|
"product": product_dict,
|
|
254
|
-
"providers": [
|
|
275
|
+
"providers": [self.get_provider_dict(product.provider)],
|
|
255
276
|
},
|
|
256
277
|
)
|
|
257
278
|
|
|
279
|
+
# add additional item props
|
|
280
|
+
for p in set(product.properties) - set(ignored_props):
|
|
281
|
+
prefix = getattr(
|
|
282
|
+
self.eodag_api.providers_config[product.provider],
|
|
283
|
+
"group",
|
|
284
|
+
product.provider,
|
|
285
|
+
)
|
|
286
|
+
key = p if ":" in p else f"{prefix}:{p}"
|
|
287
|
+
product_item["properties"][key] = product.properties[p]
|
|
288
|
+
|
|
258
289
|
# parse download link
|
|
259
|
-
|
|
290
|
+
downloadlink_href = (
|
|
291
|
+
f"{catalog['url']}/items/{product.properties['id']}/download"
|
|
292
|
+
)
|
|
293
|
+
_dc_qs = product.properties.get("_dc_qs")
|
|
294
|
+
url_parts = urlparse(downloadlink_href)
|
|
260
295
|
query_dict = parse_qs(url_parts.query)
|
|
261
296
|
without_arg_url = (
|
|
262
297
|
f"{url_parts.scheme}://{url_parts.netloc}{url_parts.path}"
|
|
263
298
|
if url_parts.scheme
|
|
264
299
|
else f"{url_parts.netloc}{url_parts.path}"
|
|
265
300
|
)
|
|
266
|
-
|
|
267
301
|
# add provider to query-args
|
|
268
|
-
|
|
269
|
-
|
|
302
|
+
p_config = self.eodag_api.providers_config[product.provider]
|
|
303
|
+
query_dict.update(provider=[getattr(p_config, "group", p_config.name)])
|
|
270
304
|
# add datacube query-string to query-args
|
|
271
|
-
_dc_qs = product_item["assets"]["downloadLink"].pop("_dc_qs", None)
|
|
272
305
|
if _dc_qs:
|
|
273
|
-
query_dict.update(_dc_qs=_dc_qs)
|
|
274
|
-
|
|
275
|
-
# update download link with up-to-date query-args
|
|
306
|
+
query_dict.update(_dc_qs=[_dc_qs])
|
|
276
307
|
if query_dict:
|
|
277
|
-
|
|
278
|
-
"
|
|
279
|
-
|
|
308
|
+
downloadlink_href = (
|
|
309
|
+
f"{without_arg_url}?{urlencode(query_dict, doseq=True)}"
|
|
310
|
+
)
|
|
280
311
|
|
|
281
|
-
#
|
|
282
|
-
|
|
283
|
-
|
|
284
|
-
|
|
285
|
-
for asset_key, asset_value in origin_assets.items():
|
|
286
|
-
# use origin asset as default
|
|
287
|
-
product_item["assets"][asset_key] = asset_value
|
|
288
|
-
# origin assets as alternate link
|
|
289
|
-
product_item["assets"][asset_key]["alternate"] = {
|
|
290
|
-
"origin": {
|
|
291
|
-
"title": "Origin asset link",
|
|
292
|
-
"href": asset_value["href"],
|
|
293
|
-
}
|
|
294
|
-
}
|
|
295
|
-
# use server-mode assets download links
|
|
296
|
-
asset_value["href"] = without_arg_url
|
|
297
|
-
if query_dict:
|
|
298
|
-
product_item["assets"][asset_key][
|
|
299
|
-
"href"
|
|
300
|
-
] += f"/{asset_key}?{urlencode(query_dict, doseq=True)}"
|
|
301
|
-
else:
|
|
302
|
-
product_item["assets"][asset_key]["href"] += f"/{asset_key}"
|
|
312
|
+
# generate STAC assets
|
|
313
|
+
product_item["assets"] = self._get_assets(
|
|
314
|
+
product, downloadlink_href, without_arg_url, query_dict, _dc_qs
|
|
315
|
+
)
|
|
303
316
|
|
|
304
317
|
# apply conversion if needed
|
|
305
318
|
for prop_key, prop_val in need_conversion.items():
|
|
@@ -321,16 +334,23 @@ class StacItem(StacCommon):
|
|
|
321
334
|
format_args = deepcopy(self.stac_config)
|
|
322
335
|
format_args["catalog"] = catalog
|
|
323
336
|
format_args["item"] = product_item
|
|
324
|
-
product_item
|
|
325
|
-
product_item, **format_args
|
|
326
|
-
)
|
|
337
|
+
product_item = format_dict_items(product_item, **format_args)
|
|
327
338
|
product_item["bbox"] = [float(i) for i in product_item["bbox"]]
|
|
328
339
|
|
|
340
|
+
# transform shapely geometry to geojson
|
|
341
|
+
product_item["geometry"] = geojson.loads(
|
|
342
|
+
geojson.dumps(product_item["geometry"])
|
|
343
|
+
)
|
|
344
|
+
|
|
329
345
|
# remove empty properties
|
|
330
346
|
product_item = self.__filter_item_properties_values(product_item)
|
|
331
347
|
|
|
348
|
+
# quote invalid characters in links
|
|
349
|
+
for link in product_item["links"]:
|
|
350
|
+
link["href"] = _quote_url_path(link["href"])
|
|
351
|
+
|
|
332
352
|
# update item link with datacube query-string
|
|
333
|
-
if _dc_qs:
|
|
353
|
+
if _dc_qs or self.provider:
|
|
334
354
|
url_parts = urlparse(str(product_item["links"][0]["href"]))
|
|
335
355
|
without_arg_url = (
|
|
336
356
|
f"{url_parts.scheme}://{url_parts.netloc}{url_parts.path}"
|
|
@@ -345,50 +365,128 @@ class StacItem(StacCommon):
|
|
|
345
365
|
|
|
346
366
|
return item_list
|
|
347
367
|
|
|
368
|
+
def _get_assets(
|
|
369
|
+
self,
|
|
370
|
+
product: EOProduct,
|
|
371
|
+
downloadlink_href: str,
|
|
372
|
+
without_arg_url: str,
|
|
373
|
+
query_dict: Optional[Dict[str, Any]] = None,
|
|
374
|
+
_dc_qs: Optional[str] = None,
|
|
375
|
+
) -> Dict[str, Any]:
|
|
376
|
+
assets: Dict[str, Any] = {}
|
|
377
|
+
settings = Settings.from_environment()
|
|
378
|
+
|
|
379
|
+
if _dc_qs:
|
|
380
|
+
parsed = urlparse(product.remote_location)
|
|
381
|
+
fragments = parsed.fragment.split("?")
|
|
382
|
+
parsed = parsed._replace(fragment=f"{fragments[0]}?_dc_qs={_dc_qs}")
|
|
383
|
+
origin_href = urlunparse(parsed)
|
|
384
|
+
else:
|
|
385
|
+
origin_href = product.remote_location
|
|
386
|
+
|
|
387
|
+
# update download link with up-to-date query-args
|
|
388
|
+
quoted_href = _quote_url_path(
|
|
389
|
+
downloadlink_href
|
|
390
|
+
) # quote invalid characters in url
|
|
391
|
+
assets["downloadLink"] = {
|
|
392
|
+
"title": "Download link",
|
|
393
|
+
"href": quoted_href,
|
|
394
|
+
"type": "application/zip",
|
|
395
|
+
}
|
|
396
|
+
|
|
397
|
+
if not origin_href.startswith(tuple(settings.origin_url_blacklist)):
|
|
398
|
+
assets["downloadLink"]["alternate"] = {
|
|
399
|
+
"origin": {
|
|
400
|
+
"title": "Origin asset link",
|
|
401
|
+
"href": origin_href,
|
|
402
|
+
}
|
|
403
|
+
}
|
|
404
|
+
|
|
405
|
+
if "storageStatus" in product.properties:
|
|
406
|
+
assets["downloadLink"]["storage:tier"] = product.properties["storageStatus"]
|
|
407
|
+
|
|
408
|
+
# move origin asset urls to alternate links and replace with eodag-server ones
|
|
409
|
+
if product.assets:
|
|
410
|
+
origin_assets = product.assets.as_dict()
|
|
411
|
+
# replace origin asset urls with eodag-server ones
|
|
412
|
+
for asset_key, asset_value in origin_assets.items():
|
|
413
|
+
# use origin asset as default
|
|
414
|
+
assets[asset_key] = asset_value
|
|
415
|
+
# origin assets as alternate link
|
|
416
|
+
if not asset_value["href"].startswith(
|
|
417
|
+
tuple(settings.origin_url_blacklist)
|
|
418
|
+
):
|
|
419
|
+
assets[asset_key]["alternate"] = {
|
|
420
|
+
"origin": {
|
|
421
|
+
"title": "Origin asset link",
|
|
422
|
+
"href": asset_value["href"],
|
|
423
|
+
}
|
|
424
|
+
}
|
|
425
|
+
# use server-mode assets download links
|
|
426
|
+
asset_value["href"] = without_arg_url
|
|
427
|
+
if query_dict:
|
|
428
|
+
assets[asset_key][
|
|
429
|
+
"href"
|
|
430
|
+
] += f"/{asset_key}?{urlencode(query_dict, doseq=True)}"
|
|
431
|
+
else:
|
|
432
|
+
assets[asset_key]["href"] += f"/{asset_key}"
|
|
433
|
+
if asset_type := asset_value.get("type", None):
|
|
434
|
+
assets[asset_key]["type"] = asset_type
|
|
435
|
+
if origin := assets[asset_key].get("alternate", {}).get("origin"):
|
|
436
|
+
origin["type"] = asset_type
|
|
437
|
+
asset_value["href"] = _quote_url_path(asset_value["href"])
|
|
438
|
+
|
|
439
|
+
if thumbnail_url := product.properties.get(
|
|
440
|
+
"quicklook", product.properties.get("thumbnail", None)
|
|
441
|
+
):
|
|
442
|
+
assets["thumbnail"] = {
|
|
443
|
+
"title": "Thumbnail",
|
|
444
|
+
"href": thumbnail_url,
|
|
445
|
+
"role": "thumbnail",
|
|
446
|
+
}
|
|
447
|
+
if mime_type := guess_file_type(thumbnail_url):
|
|
448
|
+
assets["thumbnail"]["type"] = mime_type
|
|
449
|
+
return assets
|
|
450
|
+
|
|
348
451
|
def get_stac_items(
|
|
349
|
-
self,
|
|
452
|
+
self,
|
|
453
|
+
search_results: SearchResult,
|
|
454
|
+
total: int,
|
|
455
|
+
catalog: Dict[str, Any],
|
|
456
|
+
next_link: Optional[Dict[str, Any]],
|
|
350
457
|
) -> Dict[str, Any]:
|
|
351
458
|
"""Build STAC items from EODAG search results
|
|
352
459
|
|
|
353
460
|
:param search_results: EODAG search results
|
|
354
|
-
:type search_results: :class:`~eodag.api.search_result.SearchResult`
|
|
355
461
|
:param catalog: STAC catalog dict used for parsing item metadata
|
|
356
|
-
:
|
|
357
|
-
:returns: Items dictionnary
|
|
358
|
-
:rtype: dict
|
|
462
|
+
:returns: Items dictionary
|
|
359
463
|
"""
|
|
360
464
|
items_model = deepcopy(self.stac_config["items"])
|
|
361
465
|
|
|
362
|
-
search_results.numberMatched = search_results.properties["totalResults"]
|
|
363
|
-
search_results.numberReturned = len(search_results)
|
|
364
|
-
|
|
365
|
-
# next page link
|
|
366
466
|
if "?" in self.url:
|
|
367
467
|
# search endpoint: use page url as self link
|
|
368
468
|
for i, _ in enumerate(items_model["links"]):
|
|
369
469
|
if items_model["links"][i]["rel"] == "self":
|
|
370
470
|
items_model["links"][i]["href"] = catalog["url"]
|
|
371
471
|
|
|
372
|
-
|
|
373
|
-
datetime.now(timezone.utc).isoformat().replace("+00:00", "") + "Z"
|
|
374
|
-
)
|
|
472
|
+
timestamp = datetime.now(timezone.utc).isoformat().replace("+00:00", "Z")
|
|
375
473
|
|
|
376
474
|
# parse jsonpath
|
|
377
475
|
items = jsonpath_parse_dict_items(
|
|
378
|
-
items_model,
|
|
476
|
+
items_model,
|
|
477
|
+
{
|
|
478
|
+
"numberMatched": total,
|
|
479
|
+
"numberReturned": len(search_results),
|
|
480
|
+
"timeStamp": timestamp,
|
|
481
|
+
},
|
|
379
482
|
)
|
|
380
483
|
# parse f-strings
|
|
381
484
|
format_args = deepcopy(self.stac_config)
|
|
382
485
|
format_args["catalog"] = catalog
|
|
383
486
|
items = format_dict_items(items, **format_args)
|
|
384
487
|
|
|
385
|
-
|
|
386
|
-
|
|
387
|
-
search_results.properties["itemsPerPage"]
|
|
388
|
-
* search_results.properties["page"]
|
|
389
|
-
>= search_results.properties["totalResults"]
|
|
390
|
-
):
|
|
391
|
-
items["links"] = [link for link in items["links"] if link["rel"] != "next"]
|
|
488
|
+
if next_link:
|
|
489
|
+
items["links"].append(next_link)
|
|
392
490
|
|
|
393
491
|
# provide static catalog to build features
|
|
394
492
|
if "search?" in catalog["url"]:
|
|
@@ -402,7 +500,7 @@ class StacItem(StacCommon):
|
|
|
402
500
|
items["features"] = self.__get_item_list(search_results, catalog)
|
|
403
501
|
|
|
404
502
|
self.update_data(items)
|
|
405
|
-
return
|
|
503
|
+
return self.data
|
|
406
504
|
|
|
407
505
|
def __filter_item_model_properties(
|
|
408
506
|
self, item_model: Dict[str, Any], product_type: str
|
|
@@ -412,11 +510,8 @@ class StacItem(StacCommon):
|
|
|
412
510
|
part of oseo extension.
|
|
413
511
|
|
|
414
512
|
:param item_model: Item model from stac_config
|
|
415
|
-
:type item_model: dict
|
|
416
513
|
:param product_type: Product type
|
|
417
|
-
:type product_type: str
|
|
418
514
|
:returns: Filtered item model
|
|
419
|
-
:rtype: dict
|
|
420
515
|
"""
|
|
421
516
|
try:
|
|
422
517
|
product_type_dict = [
|
|
@@ -427,12 +522,10 @@ class StacItem(StacCommon):
|
|
|
427
522
|
if pt["ID"] == product_type
|
|
428
523
|
or ("alias" in pt and pt["alias"] == product_type)
|
|
429
524
|
][0]
|
|
430
|
-
except IndexError:
|
|
525
|
+
except IndexError as e:
|
|
431
526
|
raise NoMatchingProductType(
|
|
432
|
-
"Product type {} not available for {}"
|
|
433
|
-
|
|
434
|
-
)
|
|
435
|
-
)
|
|
527
|
+
f"Product type {product_type} not available for {self.provider}"
|
|
528
|
+
) from e
|
|
436
529
|
|
|
437
530
|
result_item_model = deepcopy(item_model)
|
|
438
531
|
result_item_model["stac_extensions"] = list(
|
|
@@ -479,9 +572,7 @@ class StacItem(StacCommon):
|
|
|
479
572
|
"""Removes empty properties, unused extensions, and add missing extensions
|
|
480
573
|
|
|
481
574
|
:param item: STAC item data
|
|
482
|
-
:type item: dict
|
|
483
575
|
:returns: Filtered item model
|
|
484
|
-
:rtype: dict
|
|
485
576
|
"""
|
|
486
577
|
all_extensions_dict: Dict[str, str] = deepcopy(
|
|
487
578
|
self.stac_config["stac_extensions"]
|
|
@@ -512,21 +603,13 @@ class StacItem(StacCommon):
|
|
|
512
603
|
"""Build STAC item from EODAG product
|
|
513
604
|
|
|
514
605
|
:param product: EODAG product
|
|
515
|
-
:type product: :class:`eodag.api.product._product.EOProduct`
|
|
516
606
|
:returns: STAC item
|
|
517
|
-
:rtype: list
|
|
518
607
|
"""
|
|
519
608
|
product_type = str(product.product_type)
|
|
520
609
|
|
|
521
610
|
item_model = self.__filter_item_model_properties(
|
|
522
611
|
self.stac_config["item"], product_type
|
|
523
612
|
)
|
|
524
|
-
provider_model = deepcopy(self.stac_config["provider"])
|
|
525
|
-
|
|
526
|
-
provider_dict = jsonpath_parse_dict_items(
|
|
527
|
-
provider_model,
|
|
528
|
-
{"provider": self.eodag_api.providers_config[product.provider].__dict__},
|
|
529
|
-
)
|
|
530
613
|
|
|
531
614
|
catalog = StacCatalog(
|
|
532
615
|
url=self.url.split("/items")[0],
|
|
@@ -534,7 +617,7 @@ class StacItem(StacCommon):
|
|
|
534
617
|
root=self.root,
|
|
535
618
|
provider=self.provider,
|
|
536
619
|
eodag_api=self.eodag_api,
|
|
537
|
-
|
|
620
|
+
collection=product_type,
|
|
538
621
|
)
|
|
539
622
|
|
|
540
623
|
product_dict = deepcopy(product.__dict__)
|
|
@@ -545,14 +628,15 @@ class StacItem(StacCommon):
|
|
|
545
628
|
item_model,
|
|
546
629
|
{
|
|
547
630
|
"product": product_dict,
|
|
548
|
-
"providers":
|
|
631
|
+
"providers": [self.get_provider_dict(product.provider)],
|
|
549
632
|
},
|
|
550
633
|
)
|
|
551
634
|
# parse f-strings
|
|
552
635
|
format_args = deepcopy(self.stac_config)
|
|
553
|
-
format_args["catalog"] =
|
|
554
|
-
|
|
555
|
-
|
|
636
|
+
format_args["catalog"] = {
|
|
637
|
+
**catalog.data,
|
|
638
|
+
**{"url": catalog.url, "root": catalog.root},
|
|
639
|
+
}
|
|
556
640
|
format_args["item"] = product_item
|
|
557
641
|
product_item = format_dict_items(product_item, **format_args)
|
|
558
642
|
product_item["bbox"] = [float(i) for i in product_item["bbox"]]
|
|
@@ -561,24 +645,46 @@ class StacItem(StacCommon):
|
|
|
561
645
|
product_item = self.__filter_item_properties_values(product_item)
|
|
562
646
|
|
|
563
647
|
self.update_data(product_item)
|
|
564
|
-
return self.
|
|
648
|
+
return self.data
|
|
565
649
|
|
|
566
650
|
|
|
567
651
|
class StacCollection(StacCommon):
|
|
568
652
|
"""Stac collection object
|
|
569
653
|
|
|
570
654
|
:param url: Requested URL
|
|
571
|
-
:type url: str
|
|
572
655
|
:param stac_config: STAC configuration from stac.yml conf file
|
|
573
|
-
:type stac_config: dict
|
|
574
656
|
:param provider: (optional) Chosen provider
|
|
575
|
-
:type provider: str
|
|
576
657
|
:param eodag_api: EODAG python API instance
|
|
577
|
-
:type eodag_api: :class:`eodag.api.core.EODataAccessGateway`
|
|
578
658
|
:param root: (optional) API root
|
|
579
|
-
:type root: str
|
|
580
659
|
"""
|
|
581
660
|
|
|
661
|
+
# External STAC collections
|
|
662
|
+
ext_stac_collections: Dict[str, Dict[str, Any]] = dict()
|
|
663
|
+
|
|
664
|
+
@classmethod
|
|
665
|
+
def fetch_external_stac_collections(cls, eodag_api: EODataAccessGateway) -> None:
|
|
666
|
+
"""Load external STAC collections
|
|
667
|
+
|
|
668
|
+
:param eodag_api: EODAG python API instance
|
|
669
|
+
"""
|
|
670
|
+
list_product_types = eodag_api.list_product_types(fetch_providers=False)
|
|
671
|
+
for product_type in list_product_types:
|
|
672
|
+
ext_stac_collection_path = product_type.get("stacCollection")
|
|
673
|
+
if not ext_stac_collection_path:
|
|
674
|
+
continue
|
|
675
|
+
logger.info(f"Fetching external STAC collection for {product_type['ID']}")
|
|
676
|
+
|
|
677
|
+
try:
|
|
678
|
+
ext_stac_collection = fetch_json(ext_stac_collection_path)
|
|
679
|
+
except (RequestError, TimeOutError) as e:
|
|
680
|
+
logger.debug(e)
|
|
681
|
+
logger.warning(
|
|
682
|
+
f"Could not read remote external STAC collection from {ext_stac_collection_path}",
|
|
683
|
+
)
|
|
684
|
+
ext_stac_collection = {}
|
|
685
|
+
|
|
686
|
+
cls.ext_stac_collections[product_type["ID"]] = ext_stac_collection
|
|
687
|
+
|
|
582
688
|
def __init__(
|
|
583
689
|
self,
|
|
584
690
|
url: str,
|
|
@@ -595,165 +701,174 @@ class StacCollection(StacCommon):
|
|
|
595
701
|
root=root,
|
|
596
702
|
)
|
|
597
703
|
|
|
598
|
-
def
|
|
599
|
-
|
|
600
|
-
) -> List[Dict[str, Any]]:
|
|
601
|
-
"""Returns a list of supported product types
|
|
704
|
+
def __list_product_type_providers(self, product_type: Dict[str, Any]) -> List[str]:
|
|
705
|
+
"""Retrieve a list of providers for a given product type.
|
|
602
706
|
|
|
603
|
-
:param
|
|
604
|
-
:
|
|
605
|
-
:returns: List of corresponding product types
|
|
606
|
-
:rtype: list
|
|
707
|
+
:param product_type: Dictionary containing information about the product type.
|
|
708
|
+
:return: A list of provider names.
|
|
607
709
|
"""
|
|
608
|
-
if
|
|
609
|
-
|
|
610
|
-
try:
|
|
611
|
-
guessed_product_types = self.eodag_api.guess_product_type(**filters)
|
|
612
|
-
except NoMatchingProductType:
|
|
613
|
-
guessed_product_types = []
|
|
614
|
-
if guessed_product_types:
|
|
615
|
-
product_types = [
|
|
616
|
-
pt
|
|
617
|
-
for pt in self.eodag_api.list_product_types(provider=self.provider)
|
|
618
|
-
if pt["ID"] in guessed_product_types
|
|
619
|
-
]
|
|
620
|
-
else:
|
|
621
|
-
product_types = self.eodag_api.list_product_types(provider=self.provider)
|
|
622
|
-
return product_types
|
|
710
|
+
if self.provider:
|
|
711
|
+
return [self.provider]
|
|
623
712
|
|
|
624
|
-
|
|
625
|
-
|
|
626
|
-
|
|
627
|
-
|
|
713
|
+
return [
|
|
714
|
+
plugin.provider
|
|
715
|
+
for plugin in self.eodag_api._plugins_manager.get_search_plugins(
|
|
716
|
+
product_type=product_type.get("_id", product_type["ID"])
|
|
717
|
+
)
|
|
718
|
+
]
|
|
628
719
|
|
|
629
|
-
|
|
630
|
-
:
|
|
631
|
-
|
|
632
|
-
|
|
720
|
+
def __generate_stac_collection(
|
|
721
|
+
self, collection_model: Any, product_type: Dict[str, Any]
|
|
722
|
+
) -> Dict[str, Any]:
|
|
723
|
+
"""Generate a STAC collection dictionary for a given product type.
|
|
724
|
+
|
|
725
|
+
:param collection_model: The base model for the STAC collection.
|
|
726
|
+
:param product_type: Dictionary containing information about the product type.
|
|
727
|
+
:return: A dictionary representing the STAC collection for the given product type.
|
|
633
728
|
"""
|
|
634
|
-
|
|
635
|
-
provider_model = deepcopy(self.stac_config["provider"])
|
|
729
|
+
providers = self.__list_product_type_providers(product_type)
|
|
636
730
|
|
|
637
|
-
|
|
731
|
+
providers_dict: Dict[str, Dict[str, Any]] = {}
|
|
732
|
+
for provider in providers:
|
|
733
|
+
p_dict = self.get_provider_dict(provider)
|
|
734
|
+
providers_dict.setdefault(p_dict["name"], p_dict)
|
|
735
|
+
providers_list = list(providers_dict.values())
|
|
638
736
|
|
|
639
|
-
|
|
640
|
-
|
|
641
|
-
|
|
642
|
-
|
|
643
|
-
|
|
644
|
-
|
|
645
|
-
|
|
646
|
-
|
|
647
|
-
|
|
648
|
-
|
|
649
|
-
|
|
650
|
-
|
|
651
|
-
)
|
|
652
|
-
]
|
|
653
|
-
providers_models: List[Dict[str, Any]] = []
|
|
654
|
-
for provider in providers:
|
|
655
|
-
provider_m = jsonpath_parse_dict_items(
|
|
656
|
-
provider_model,
|
|
657
|
-
{"provider": self.eodag_api.providers_config[provider].__dict__},
|
|
658
|
-
)
|
|
659
|
-
providers_models.append(provider_m)
|
|
737
|
+
# parse jsonpath
|
|
738
|
+
product_type_collection = jsonpath_parse_dict_items(
|
|
739
|
+
collection_model,
|
|
740
|
+
{
|
|
741
|
+
"product_type": product_type,
|
|
742
|
+
"providers": providers_list,
|
|
743
|
+
},
|
|
744
|
+
)
|
|
745
|
+
# override EODAG's collection with the external collection
|
|
746
|
+
ext_stac_collection = deepcopy(
|
|
747
|
+
self.ext_stac_collections.get(product_type["ID"], {})
|
|
748
|
+
)
|
|
660
749
|
|
|
661
|
-
|
|
662
|
-
|
|
663
|
-
|
|
664
|
-
|
|
665
|
-
|
|
666
|
-
|
|
667
|
-
|
|
668
|
-
|
|
669
|
-
|
|
670
|
-
|
|
671
|
-
|
|
672
|
-
|
|
673
|
-
**
|
|
750
|
+
# update links (keep eodag links as defaults)
|
|
751
|
+
ext_stac_collection.setdefault("links", {})
|
|
752
|
+
for link in product_type_collection["links"]:
|
|
753
|
+
ext_stac_collection["links"] = [
|
|
754
|
+
x for x in ext_stac_collection["links"] if x["rel"] != link["rel"]
|
|
755
|
+
]
|
|
756
|
+
ext_stac_collection["links"].append(link)
|
|
757
|
+
|
|
758
|
+
# merge "summaries"
|
|
759
|
+
ext_stac_collection["summaries"] = {
|
|
760
|
+
k: v
|
|
761
|
+
for k, v in {
|
|
762
|
+
**ext_stac_collection.get("summaries", {}),
|
|
763
|
+
**product_type_collection["summaries"],
|
|
764
|
+
}.items()
|
|
765
|
+
if v and any(v)
|
|
766
|
+
}
|
|
767
|
+
|
|
768
|
+
# merge "keywords" lists
|
|
769
|
+
try:
|
|
770
|
+
ext_stac_collection["keywords"] = [
|
|
771
|
+
k
|
|
772
|
+
for k in set(
|
|
773
|
+
ext_stac_collection.get("keywords", [])
|
|
774
|
+
+ product_type_collection["keywords"]
|
|
775
|
+
)
|
|
776
|
+
if k is not None
|
|
777
|
+
]
|
|
778
|
+
except TypeError as e:
|
|
779
|
+
logger.warning(
|
|
780
|
+
f"Could not merge keywords from external collection for {product_type['ID']}: {str(e)}"
|
|
674
781
|
)
|
|
675
|
-
|
|
676
|
-
|
|
782
|
+
logger.debug(
|
|
783
|
+
f"External collection keywords: {str(ext_stac_collection.get('keywords'))}, ",
|
|
784
|
+
f"Product type keywords: {str(product_type_collection['keywords'])}",
|
|
677
785
|
)
|
|
678
786
|
|
|
679
|
-
|
|
787
|
+
# merge providers
|
|
788
|
+
if "providers" in ext_stac_collection:
|
|
789
|
+
ext_stac_collection["providers"] += product_type_collection["providers"]
|
|
680
790
|
|
|
681
|
-
|
|
682
|
-
|
|
683
|
-
def get_collections(
|
|
684
|
-
self, filters: Optional[Dict[str, Any]] = None
|
|
685
|
-
) -> Dict[str, Any]:
|
|
686
|
-
"""Build STAC collections
|
|
687
|
-
|
|
688
|
-
:param filters: (optional) Additional filters for collections search
|
|
689
|
-
:type filters: dict
|
|
690
|
-
:returns: Collections dictionnary
|
|
691
|
-
:rtype: dict
|
|
692
|
-
"""
|
|
693
|
-
collections = deepcopy(self.stac_config["collections"])
|
|
694
|
-
collections["collections"] = self.__get_collection_list(filters)
|
|
791
|
+
product_type_collection.update(ext_stac_collection)
|
|
695
792
|
|
|
696
|
-
#
|
|
793
|
+
# parse f-strings
|
|
697
794
|
format_args = deepcopy(self.stac_config)
|
|
698
|
-
format_args["
|
|
795
|
+
format_args["collection"] = {
|
|
796
|
+
**product_type_collection,
|
|
797
|
+
**{
|
|
798
|
+
"url": self.url
|
|
799
|
+
if self.url.endswith(product_type["ID"])
|
|
800
|
+
else f"{self.url}/{product_type['ID']}",
|
|
801
|
+
"root": self.root,
|
|
802
|
+
},
|
|
803
|
+
}
|
|
804
|
+
product_type_collection = format_dict_items(
|
|
805
|
+
product_type_collection, **format_args
|
|
806
|
+
)
|
|
699
807
|
|
|
700
|
-
|
|
701
|
-
format_dict_items(link, **format_args) for link in collections["links"]
|
|
702
|
-
]
|
|
808
|
+
return product_type_collection
|
|
703
809
|
|
|
704
|
-
|
|
705
|
-
|
|
706
|
-
|
|
707
|
-
|
|
708
|
-
|
|
709
|
-
|
|
710
|
-
|
|
711
|
-
|
|
712
|
-
|
|
713
|
-
|
|
810
|
+
def get_collection_list(
|
|
811
|
+
self,
|
|
812
|
+
collection: Optional[str] = None,
|
|
813
|
+
q: Optional[str] = None,
|
|
814
|
+
platform: Optional[str] = None,
|
|
815
|
+
instrument: Optional[str] = None,
|
|
816
|
+
constellation: Optional[str] = None,
|
|
817
|
+
datetime: Optional[str] = None,
|
|
818
|
+
) -> List[Dict[str, Any]]:
|
|
819
|
+
"""Build STAC collections list
|
|
714
820
|
|
|
715
|
-
|
|
716
|
-
|
|
821
|
+
:param filters: (optional) Additional filters for collections search
|
|
822
|
+
:returns: STAC collection dicts list
|
|
823
|
+
"""
|
|
824
|
+
collection_model = deepcopy(self.stac_config["collection"])
|
|
717
825
|
|
|
718
|
-
|
|
719
|
-
"""Build STAC collection by its id
|
|
826
|
+
start, end = str_to_interval(datetime)
|
|
720
827
|
|
|
721
|
-
|
|
722
|
-
|
|
723
|
-
:returns: Collection dictionary
|
|
724
|
-
:rtype: dict
|
|
725
|
-
"""
|
|
726
|
-
collection_list = self.__get_collection_list(
|
|
727
|
-
filters={"productType": collection_id}
|
|
828
|
+
all_pt = self.eodag_api.list_product_types(
|
|
829
|
+
provider=self.provider, fetch_providers=False
|
|
728
830
|
)
|
|
729
831
|
|
|
730
|
-
|
|
731
|
-
|
|
732
|
-
|
|
733
|
-
|
|
832
|
+
if any((collection, q, platform, instrument, constellation, datetime)):
|
|
833
|
+
try:
|
|
834
|
+
guessed_product_types = self.eodag_api.guess_product_type(
|
|
835
|
+
free_text=q,
|
|
836
|
+
platformSerialIdentifier=platform,
|
|
837
|
+
instrument=instrument,
|
|
838
|
+
platform=constellation,
|
|
839
|
+
productType=collection,
|
|
840
|
+
missionStartDate=start.isoformat() if start else None,
|
|
841
|
+
missionEndDate=end.isoformat() if end else None,
|
|
842
|
+
)
|
|
843
|
+
except NoMatchingProductType:
|
|
844
|
+
product_types = []
|
|
845
|
+
else:
|
|
846
|
+
product_types = [
|
|
847
|
+
pt for pt in all_pt if pt["ID"] in guessed_product_types
|
|
848
|
+
]
|
|
849
|
+
else:
|
|
850
|
+
product_types = all_pt
|
|
734
851
|
|
|
735
|
-
|
|
736
|
-
|
|
852
|
+
# list product types with all metadata using guessed ids
|
|
853
|
+
collection_list: List[Dict[str, Any]] = []
|
|
854
|
+
for product_type in product_types:
|
|
855
|
+
stac_collection = self.__generate_stac_collection(
|
|
856
|
+
collection_model, product_type
|
|
857
|
+
)
|
|
858
|
+
collection_list.append(stac_collection)
|
|
859
|
+
|
|
860
|
+
return collection_list
|
|
737
861
|
|
|
738
862
|
|
|
739
863
|
class StacCatalog(StacCommon):
|
|
740
864
|
"""Stac Catalog object
|
|
741
865
|
|
|
742
866
|
:param url: Requested URL
|
|
743
|
-
:type url: str
|
|
744
867
|
:param stac_config: STAC configuration from stac.yml conf file
|
|
745
|
-
:type stac_config: dict
|
|
746
868
|
:param provider: Chosen provider
|
|
747
|
-
:type provider: (optional) str
|
|
748
869
|
:param eodag_api: EODAG python API instance
|
|
749
|
-
:type eodag_api: :class:`eodag.api.core.EODataAccessGateway`
|
|
750
870
|
:param root: (optional) API root
|
|
751
|
-
:
|
|
752
|
-
:param catalogs: (optional) Catalogs list
|
|
753
|
-
:type catalogs: list
|
|
754
|
-
:param fetch_providers: (optional) Whether to fetch providers for new product
|
|
755
|
-
types or not
|
|
756
|
-
:type fetch_providers: bool
|
|
871
|
+
:param collection: (optional) product type id
|
|
757
872
|
"""
|
|
758
873
|
|
|
759
874
|
def __init__(
|
|
@@ -763,8 +878,7 @@ class StacCatalog(StacCommon):
|
|
|
763
878
|
provider: Optional[str],
|
|
764
879
|
eodag_api: EODataAccessGateway,
|
|
765
880
|
root: str = "/",
|
|
766
|
-
|
|
767
|
-
fetch_providers: bool = True,
|
|
881
|
+
collection: Optional[str] = None,
|
|
768
882
|
) -> None:
|
|
769
883
|
super(StacCatalog, self).__init__(
|
|
770
884
|
url=url,
|
|
@@ -791,13 +905,12 @@ class StacCatalog(StacCommon):
|
|
|
791
905
|
self.data["links"] += self.children
|
|
792
906
|
|
|
793
907
|
# build catalog
|
|
794
|
-
self.__build_stac_catalog(
|
|
908
|
+
self.__build_stac_catalog(collection)
|
|
795
909
|
|
|
796
910
|
def __update_data_from_catalog_config(self, catalog_config: Dict[str, Any]) -> bool:
|
|
797
911
|
"""Updates configuration and data using given input catalog config
|
|
798
912
|
|
|
799
913
|
:param catalog_config: Catalog config, from yml stac_config[catalogs]
|
|
800
|
-
:type catalog_config: dict
|
|
801
914
|
"""
|
|
802
915
|
model = catalog_config["model"]
|
|
803
916
|
|
|
@@ -817,40 +930,66 @@ class StacCatalog(StacCommon):
|
|
|
817
930
|
|
|
818
931
|
return True
|
|
819
932
|
|
|
820
|
-
def
|
|
821
|
-
"""
|
|
933
|
+
def __build_stac_catalog(self, collection: Optional[str] = None) -> StacCatalog:
|
|
934
|
+
"""Build nested catalog from catalag list
|
|
822
935
|
|
|
823
|
-
:param
|
|
824
|
-
:
|
|
936
|
+
:param collection: (optional) product type id
|
|
937
|
+
:returns: This catalog obj
|
|
825
938
|
"""
|
|
826
|
-
|
|
827
|
-
|
|
828
|
-
|
|
829
|
-
|
|
830
|
-
|
|
831
|
-
|
|
939
|
+
settings = Settings.from_environment()
|
|
940
|
+
|
|
941
|
+
if not collection:
|
|
942
|
+
# Build root catalog combined with landing page
|
|
943
|
+
self.__update_data_from_catalog_config(
|
|
944
|
+
{
|
|
945
|
+
"model": {
|
|
946
|
+
**deepcopy(self.stac_config["landing_page"]),
|
|
947
|
+
**{
|
|
948
|
+
"provider": self.provider,
|
|
949
|
+
"id": settings.stac_api_landing_id,
|
|
950
|
+
"title": settings.stac_api_title,
|
|
951
|
+
"description": settings.stac_api_description,
|
|
952
|
+
},
|
|
953
|
+
}
|
|
954
|
+
}
|
|
955
|
+
)
|
|
956
|
+
else:
|
|
957
|
+
self.set_stac_product_type_by_id(collection)
|
|
958
|
+
return self
|
|
832
959
|
|
|
833
960
|
def set_stac_product_type_by_id(
|
|
834
|
-
self, product_type: str, **
|
|
961
|
+
self, product_type: str, **_: Any
|
|
835
962
|
) -> Dict[str, Any]:
|
|
836
963
|
"""Updates catalog with given product_type
|
|
837
964
|
|
|
838
965
|
:param product_type: Product type
|
|
839
|
-
:type product_type: str
|
|
840
966
|
"""
|
|
841
|
-
|
|
967
|
+
collections = StacCollection(
|
|
842
968
|
url=self.url,
|
|
843
969
|
stac_config=self.stac_config,
|
|
844
970
|
provider=self.provider,
|
|
845
971
|
eodag_api=self.eodag_api,
|
|
846
972
|
root=self.root,
|
|
847
|
-
).
|
|
848
|
-
|
|
849
|
-
|
|
973
|
+
).get_collection_list(collection=product_type)
|
|
974
|
+
|
|
975
|
+
if not collections:
|
|
976
|
+
raise NotAvailableError(f"Collection {product_type} does not exist.")
|
|
977
|
+
|
|
978
|
+
cat_model = {
|
|
979
|
+
"id": "{collection[id]}",
|
|
980
|
+
"title": "{collection[title]}",
|
|
981
|
+
"description": "{collection[description]}",
|
|
982
|
+
"extent": "{collection[extent]}",
|
|
983
|
+
"crs": "http://www.opengis.net/def/crs/OGC/1.3/CRS84",
|
|
984
|
+
"keywords": "{collection[keywords]}",
|
|
985
|
+
"license": "{collection[license]}",
|
|
986
|
+
"providers": "{collection[providers]}",
|
|
987
|
+
"summaries": "{collection[summaries]}",
|
|
988
|
+
}
|
|
850
989
|
# parse f-strings
|
|
851
990
|
format_args = deepcopy(self.stac_config)
|
|
852
991
|
format_args["catalog"] = defaultdict(str, **self.data)
|
|
853
|
-
format_args["collection"] =
|
|
992
|
+
format_args["collection"] = collections[0]
|
|
854
993
|
try:
|
|
855
994
|
parsed_dict: Dict[str, Any] = format_dict_items(cat_model, **format_args)
|
|
856
995
|
except Exception:
|
|
@@ -860,539 +999,6 @@ class StacCatalog(StacCommon):
|
|
|
860
999
|
self.update_data(parsed_dict)
|
|
861
1000
|
|
|
862
1001
|
# update search args
|
|
863
|
-
self.search_args.update({"
|
|
1002
|
+
self.search_args.update({"productType": product_type})
|
|
864
1003
|
|
|
865
1004
|
return parsed_dict
|
|
866
|
-
|
|
867
|
-
# get / set dates filters -------------------------------------------------
|
|
868
|
-
|
|
869
|
-
def get_stac_years_list(self, **kwargs: Any) -> List[int]:
|
|
870
|
-
"""Get catalog available years list
|
|
871
|
-
|
|
872
|
-
:returns: Years list
|
|
873
|
-
:rtype: list
|
|
874
|
-
"""
|
|
875
|
-
extent_date_min, extent_date_max = self.get_datetime_extent()
|
|
876
|
-
|
|
877
|
-
return list(range(extent_date_min.year, extent_date_max.year + 1))
|
|
878
|
-
|
|
879
|
-
def get_stac_months_list(self, **kwargs: Any) -> List[int]:
|
|
880
|
-
"""Get catalog available months list
|
|
881
|
-
|
|
882
|
-
:returns: Months list
|
|
883
|
-
:rtype: list
|
|
884
|
-
"""
|
|
885
|
-
extent_date_min, extent_date_max = self.get_datetime_extent()
|
|
886
|
-
|
|
887
|
-
return list(
|
|
888
|
-
range(
|
|
889
|
-
extent_date_min.month,
|
|
890
|
-
(extent_date_max - relativedelta(days=1)).month + 1,
|
|
891
|
-
)
|
|
892
|
-
)
|
|
893
|
-
|
|
894
|
-
def get_stac_days_list(self, **kwargs: Any) -> List[int]:
|
|
895
|
-
"""Get catalog available days list
|
|
896
|
-
|
|
897
|
-
:returns: Days list
|
|
898
|
-
:rtype: list
|
|
899
|
-
"""
|
|
900
|
-
extent_date_min, extent_date_max = self.get_datetime_extent()
|
|
901
|
-
|
|
902
|
-
return list(
|
|
903
|
-
range(
|
|
904
|
-
extent_date_min.day, (extent_date_max - relativedelta(days=1)).day + 1
|
|
905
|
-
)
|
|
906
|
-
)
|
|
907
|
-
|
|
908
|
-
def set_stac_year_by_id(self, year: str, **kwargs: Any) -> Dict[str, Any]:
|
|
909
|
-
"""Updates and returns catalog with given year
|
|
910
|
-
|
|
911
|
-
:param year: Year number
|
|
912
|
-
:type year: str
|
|
913
|
-
:returns: Updated catalog
|
|
914
|
-
:rtype: dict
|
|
915
|
-
"""
|
|
916
|
-
extent_date_min, extent_date_max = self.get_datetime_extent()
|
|
917
|
-
|
|
918
|
-
datetime_min = max(
|
|
919
|
-
[extent_date_min, dateutil.parser.parse("{}-01-01T00:00:00Z".format(year))]
|
|
920
|
-
)
|
|
921
|
-
datetime_max = min(
|
|
922
|
-
[
|
|
923
|
-
extent_date_max,
|
|
924
|
-
dateutil.parser.parse("{}-01-01T00:00:00Z".format((year)))
|
|
925
|
-
+ relativedelta(years=1),
|
|
926
|
-
]
|
|
927
|
-
)
|
|
928
|
-
|
|
929
|
-
catalog_model = deepcopy(self.stac_config["catalogs"]["year"]["model"])
|
|
930
|
-
|
|
931
|
-
parsed_dict = self.set_stac_date(datetime_min, datetime_max, catalog_model)
|
|
932
|
-
|
|
933
|
-
return parsed_dict
|
|
934
|
-
|
|
935
|
-
def set_stac_month_by_id(self, month: str, **kwargs: Any) -> Dict[str, Any]:
|
|
936
|
-
"""Updates and returns catalog with given month
|
|
937
|
-
|
|
938
|
-
:param month: Month number
|
|
939
|
-
:type month: str
|
|
940
|
-
:returns: Updated catalog
|
|
941
|
-
:rtype: dict
|
|
942
|
-
"""
|
|
943
|
-
extent_date_min, extent_date_max = self.get_datetime_extent()
|
|
944
|
-
year = extent_date_min.year
|
|
945
|
-
|
|
946
|
-
datetime_min = max(
|
|
947
|
-
[
|
|
948
|
-
extent_date_min,
|
|
949
|
-
dateutil.parser.parse("{}-{}-01T00:00:00Z".format(year, month)),
|
|
950
|
-
]
|
|
951
|
-
)
|
|
952
|
-
datetime_max = min(
|
|
953
|
-
[
|
|
954
|
-
extent_date_max,
|
|
955
|
-
dateutil.parser.parse("{}-{}-01T00:00:00Z".format(year, month))
|
|
956
|
-
+ relativedelta(months=1),
|
|
957
|
-
]
|
|
958
|
-
)
|
|
959
|
-
|
|
960
|
-
catalog_model = deepcopy(self.stac_config["catalogs"]["month"]["model"])
|
|
961
|
-
|
|
962
|
-
parsed_dict = self.set_stac_date(datetime_min, datetime_max, catalog_model)
|
|
963
|
-
|
|
964
|
-
return parsed_dict
|
|
965
|
-
|
|
966
|
-
def set_stac_day_by_id(self, day: str, **kwargs: Any) -> Dict[str, Any]:
|
|
967
|
-
"""Updates and returns catalog with given day
|
|
968
|
-
|
|
969
|
-
:param day: Day number
|
|
970
|
-
:type day: str
|
|
971
|
-
:returns: Updated catalog
|
|
972
|
-
:rtype: dict
|
|
973
|
-
"""
|
|
974
|
-
extent_date_min, extent_date_max = self.get_datetime_extent()
|
|
975
|
-
year = extent_date_min.year
|
|
976
|
-
month = extent_date_min.month
|
|
977
|
-
|
|
978
|
-
datetime_min = max(
|
|
979
|
-
[
|
|
980
|
-
extent_date_min,
|
|
981
|
-
dateutil.parser.parse("{}-{}-{}T00:00:00Z".format(year, month, day)),
|
|
982
|
-
]
|
|
983
|
-
)
|
|
984
|
-
datetime_max = min(
|
|
985
|
-
[
|
|
986
|
-
extent_date_max,
|
|
987
|
-
dateutil.parser.parse("{}-{}-{}T00:00:00Z".format(year, month, day))
|
|
988
|
-
+ relativedelta(days=1),
|
|
989
|
-
]
|
|
990
|
-
)
|
|
991
|
-
|
|
992
|
-
catalog_model = deepcopy(self.stac_config["catalogs"]["day"]["model"])
|
|
993
|
-
|
|
994
|
-
parsed_dict = self.set_stac_date(datetime_min, datetime_max, catalog_model)
|
|
995
|
-
|
|
996
|
-
return parsed_dict
|
|
997
|
-
|
|
998
|
-
def get_datetime_extent(self) -> Tuple[datetime, datetime]:
|
|
999
|
-
"""Returns catalog temporal extent as datetime objs
|
|
1000
|
-
|
|
1001
|
-
:returns: Start & stop dates
|
|
1002
|
-
:rtype: tuple
|
|
1003
|
-
"""
|
|
1004
|
-
extent_date_min = dateutil.parser.parse(DEFAULT_MISSION_START_DATE).replace(
|
|
1005
|
-
tzinfo=tz.UTC
|
|
1006
|
-
)
|
|
1007
|
-
extent_date_max = datetime.now(timezone.utc).replace(tzinfo=tz.UTC)
|
|
1008
|
-
for interval in self.data["extent"]["temporal"]["interval"]:
|
|
1009
|
-
extent_date_min_str, extent_date_max_str = interval
|
|
1010
|
-
# date min
|
|
1011
|
-
if extent_date_min_str:
|
|
1012
|
-
extent_date_min = max(
|
|
1013
|
-
extent_date_min, dateutil.parser.parse(extent_date_min_str)
|
|
1014
|
-
)
|
|
1015
|
-
# date max
|
|
1016
|
-
if extent_date_max_str:
|
|
1017
|
-
extent_date_max = min(
|
|
1018
|
-
extent_date_max, dateutil.parser.parse(extent_date_max_str)
|
|
1019
|
-
)
|
|
1020
|
-
|
|
1021
|
-
return (
|
|
1022
|
-
extent_date_min.replace(tzinfo=tz.UTC),
|
|
1023
|
-
extent_date_max.replace(tzinfo=tz.UTC),
|
|
1024
|
-
)
|
|
1025
|
-
|
|
1026
|
-
def set_stac_date(
|
|
1027
|
-
self,
|
|
1028
|
-
datetime_min: datetime,
|
|
1029
|
-
datetime_max: datetime,
|
|
1030
|
-
catalog_model: Dict[str, Any],
|
|
1031
|
-
):
|
|
1032
|
-
"""Updates catalog data using given dates
|
|
1033
|
-
|
|
1034
|
-
:param datetime_min: Date min of interval
|
|
1035
|
-
:type datetime_min: :class:`datetime`
|
|
1036
|
-
:param datetime_max: Date max of interval
|
|
1037
|
-
:type datetime_max: :class:`datetime`
|
|
1038
|
-
:param catalog_model: Catalog model to use, from yml stac_config[catalogs]
|
|
1039
|
-
:type catalog_model: dict
|
|
1040
|
-
:returns: Updated catalog
|
|
1041
|
-
:rtype: dict
|
|
1042
|
-
"""
|
|
1043
|
-
# parse f-strings
|
|
1044
|
-
format_args = deepcopy(self.stac_config)
|
|
1045
|
-
format_args["catalog"] = defaultdict(str, **self.data)
|
|
1046
|
-
format_args["date"] = defaultdict(
|
|
1047
|
-
str,
|
|
1048
|
-
{
|
|
1049
|
-
"year": datetime_min.year,
|
|
1050
|
-
"month": datetime_min.month,
|
|
1051
|
-
"day": datetime_min.day,
|
|
1052
|
-
"min": datetime_min.isoformat().replace("+00:00", "") + "Z",
|
|
1053
|
-
"max": datetime_max.isoformat().replace("+00:00", "") + "Z",
|
|
1054
|
-
},
|
|
1055
|
-
)
|
|
1056
|
-
parsed_dict: Dict[str, Any] = format_dict_items(catalog_model, **format_args)
|
|
1057
|
-
|
|
1058
|
-
self.update_data(parsed_dict)
|
|
1059
|
-
|
|
1060
|
-
# update search args
|
|
1061
|
-
self.search_args.update(
|
|
1062
|
-
{
|
|
1063
|
-
"dtstart": datetime_min.isoformat().split("T")[0],
|
|
1064
|
-
"dtend": datetime_max.isoformat().split("T")[0],
|
|
1065
|
-
}
|
|
1066
|
-
)
|
|
1067
|
-
return parsed_dict
|
|
1068
|
-
|
|
1069
|
-
# get / set cloud_cover filter --------------------------------------------
|
|
1070
|
-
|
|
1071
|
-
def get_stac_cloud_covers_list(self, **kwargs: Any) -> List[int]:
|
|
1072
|
-
"""Get cloud_cover list
|
|
1073
|
-
|
|
1074
|
-
:returns: cloud_cover list
|
|
1075
|
-
:rtype: list
|
|
1076
|
-
"""
|
|
1077
|
-
return list(range(0, 101, 10))
|
|
1078
|
-
|
|
1079
|
-
def set_stac_cloud_cover_by_id(
|
|
1080
|
-
self, cloud_cover: str, **kwargs: Any
|
|
1081
|
-
) -> Dict[str, Any]:
|
|
1082
|
-
"""Updates and returns catalog with given max cloud_cover
|
|
1083
|
-
|
|
1084
|
-
:param cloud_cover: Cloud_cover number
|
|
1085
|
-
:type cloud_cover: str
|
|
1086
|
-
:returns: Updated catalog
|
|
1087
|
-
:rtype: dict
|
|
1088
|
-
"""
|
|
1089
|
-
cat_model = deepcopy(self.stac_config["catalogs"]["cloud_cover"]["model"])
|
|
1090
|
-
# parse f-strings
|
|
1091
|
-
format_args = deepcopy(self.stac_config)
|
|
1092
|
-
format_args["catalog"] = defaultdict(str, **self.data)
|
|
1093
|
-
format_args["cloud_cover"] = cloud_cover
|
|
1094
|
-
parsed_dict: Dict[str, Any] = format_dict_items(cat_model, **format_args)
|
|
1095
|
-
|
|
1096
|
-
self.update_data(parsed_dict)
|
|
1097
|
-
|
|
1098
|
-
# update search args
|
|
1099
|
-
self.search_args.update({"query": {"eo:cloud_cover": {"lte": cloud_cover}}})
|
|
1100
|
-
|
|
1101
|
-
return parsed_dict
|
|
1102
|
-
|
|
1103
|
-
# get / set locations filter ----------------------------------------------
|
|
1104
|
-
|
|
1105
|
-
def get_stac_location_list(self, catalog_name: str) -> List[str]:
|
|
1106
|
-
"""Get locations list using stac_conf & locations_config
|
|
1107
|
-
|
|
1108
|
-
:param catalog_name: Catalog/location name
|
|
1109
|
-
:type catalog_name: str
|
|
1110
|
-
:returns: Locations list
|
|
1111
|
-
:rtype: list
|
|
1112
|
-
"""
|
|
1113
|
-
|
|
1114
|
-
if catalog_name not in self.stac_config["catalogs"]:
|
|
1115
|
-
logger.warning(
|
|
1116
|
-
"no entry found for {} in location_config".format(catalog_name)
|
|
1117
|
-
)
|
|
1118
|
-
return []
|
|
1119
|
-
location_config = self.stac_config["catalogs"][catalog_name]
|
|
1120
|
-
|
|
1121
|
-
for k in ["path", "attr"]:
|
|
1122
|
-
if k not in location_config.keys():
|
|
1123
|
-
logger.warning(
|
|
1124
|
-
"no {} key found for {} in location_config".format(k, catalog_name)
|
|
1125
|
-
)
|
|
1126
|
-
return []
|
|
1127
|
-
path = location_config["path"]
|
|
1128
|
-
attr = location_config["attr"]
|
|
1129
|
-
|
|
1130
|
-
with shapefile.Reader(path) as shp:
|
|
1131
|
-
countries_list: List[str] = [rec[attr] for rec in shp.records()]
|
|
1132
|
-
|
|
1133
|
-
# remove duplicates
|
|
1134
|
-
countries_list = list(set(countries_list))
|
|
1135
|
-
|
|
1136
|
-
countries_list.sort()
|
|
1137
|
-
|
|
1138
|
-
return countries_list
|
|
1139
|
-
|
|
1140
|
-
def set_stac_location_by_id(
|
|
1141
|
-
self, location: str, catalog_name: str
|
|
1142
|
-
) -> Dict[str, Any]:
|
|
1143
|
-
"""Updates and returns catalog with given location
|
|
1144
|
-
|
|
1145
|
-
:param location: Feature attribute value for shp filtering
|
|
1146
|
-
:type location: str
|
|
1147
|
-
:param catalog_name: Catalog/location name
|
|
1148
|
-
:type catalog_name: str
|
|
1149
|
-
:returns: Updated catalog
|
|
1150
|
-
:rtype: dict
|
|
1151
|
-
"""
|
|
1152
|
-
location_list_cat_key = catalog_name + "_list"
|
|
1153
|
-
|
|
1154
|
-
if location_list_cat_key not in self.stac_config["catalogs"]:
|
|
1155
|
-
logger.warning(
|
|
1156
|
-
"no entry found for {}'s list in location_config".format(catalog_name)
|
|
1157
|
-
)
|
|
1158
|
-
return {}
|
|
1159
|
-
location_config = self.stac_config["catalogs"][location_list_cat_key]
|
|
1160
|
-
|
|
1161
|
-
for k in ["path", "attr"]:
|
|
1162
|
-
if k not in location_config.keys():
|
|
1163
|
-
logger.warning(
|
|
1164
|
-
"no {} key found for {}'s list in location_config".format(
|
|
1165
|
-
k, catalog_name
|
|
1166
|
-
)
|
|
1167
|
-
)
|
|
1168
|
-
return {}
|
|
1169
|
-
path = location_config["path"]
|
|
1170
|
-
attr = location_config["attr"]
|
|
1171
|
-
|
|
1172
|
-
with shapefile.Reader(path) as shp:
|
|
1173
|
-
geom_hits = [
|
|
1174
|
-
shape(shaperec.shape)
|
|
1175
|
-
for shaperec in shp.shapeRecords()
|
|
1176
|
-
if shaperec.record.as_dict().get(attr, None) == location
|
|
1177
|
-
]
|
|
1178
|
-
|
|
1179
|
-
if len(geom_hits) == 0:
|
|
1180
|
-
logger.warning(
|
|
1181
|
-
"no feature found in %s matching %s=%s" % (path, attr, location)
|
|
1182
|
-
)
|
|
1183
|
-
return {}
|
|
1184
|
-
|
|
1185
|
-
geom = cast(BaseGeometry, unary_union(geom_hits))
|
|
1186
|
-
|
|
1187
|
-
cat_model = deepcopy(self.stac_config["catalogs"]["country"]["model"])
|
|
1188
|
-
# parse f-strings
|
|
1189
|
-
format_args = deepcopy(self.stac_config)
|
|
1190
|
-
format_args["catalog"] = defaultdict(str, **self.data)
|
|
1191
|
-
format_args["feature"] = defaultdict(str, {"geometry": geom, "id": location})
|
|
1192
|
-
parsed_dict: Dict[str, Any] = format_dict_items(cat_model, **format_args)
|
|
1193
|
-
|
|
1194
|
-
self.update_data(parsed_dict)
|
|
1195
|
-
|
|
1196
|
-
# update search args
|
|
1197
|
-
self.search_args.update({"geom": geom})
|
|
1198
|
-
|
|
1199
|
-
return parsed_dict
|
|
1200
|
-
|
|
1201
|
-
def build_locations_config(self) -> Dict[str, str]:
|
|
1202
|
-
"""Build locations config from stac_conf[locations_catalogs] & eodag_api.locations_config
|
|
1203
|
-
|
|
1204
|
-
:returns: Locations configuration dict
|
|
1205
|
-
:rtype: dict
|
|
1206
|
-
"""
|
|
1207
|
-
user_config_locations_list = self.eodag_api.locations_config
|
|
1208
|
-
|
|
1209
|
-
locations_config_model = deepcopy(self.stac_config["locations_catalogs"])
|
|
1210
|
-
|
|
1211
|
-
locations_config: Dict[str, str] = {}
|
|
1212
|
-
for loc in user_config_locations_list:
|
|
1213
|
-
# parse jsonpath
|
|
1214
|
-
parsed = jsonpath_parse_dict_items(
|
|
1215
|
-
locations_config_model, {"shp_location": loc}
|
|
1216
|
-
)
|
|
1217
|
-
|
|
1218
|
-
# set default child/parent for this location
|
|
1219
|
-
parsed["location"]["parent_key"] = "{}_list".format(loc["name"])
|
|
1220
|
-
|
|
1221
|
-
locations_config["{}_list".format(loc["name"])] = parsed["locations_list"]
|
|
1222
|
-
locations_config[loc["name"]] = parsed["location"]
|
|
1223
|
-
|
|
1224
|
-
return locations_config
|
|
1225
|
-
|
|
1226
|
-
def __build_stac_catalog(
|
|
1227
|
-
self, catalogs: List[str] = [], fetch_providers: bool = True
|
|
1228
|
-
) -> StacCatalog:
|
|
1229
|
-
"""Build nested catalog from catalag list
|
|
1230
|
-
|
|
1231
|
-
:param catalogs: (optional) Catalogs list
|
|
1232
|
-
:type catalogs: list
|
|
1233
|
-
:param fetch_providers: (optional) Whether to fetch providers for new product
|
|
1234
|
-
types or not
|
|
1235
|
-
:type fetch_providers: bool
|
|
1236
|
-
:returns: This catalog obj
|
|
1237
|
-
:rtype: :class:`eodag.stac.StacCatalog`
|
|
1238
|
-
"""
|
|
1239
|
-
# update conf with user shp locations
|
|
1240
|
-
locations_config = self.build_locations_config()
|
|
1241
|
-
|
|
1242
|
-
self.stac_config["catalogs"] = dict(
|
|
1243
|
-
deepcopy(self.stac_config["catalogs"]), **locations_config
|
|
1244
|
-
)
|
|
1245
|
-
|
|
1246
|
-
if len(catalogs) == 0:
|
|
1247
|
-
# Build root catalog combined with landing page
|
|
1248
|
-
self.__update_data_from_catalog_config(
|
|
1249
|
-
{
|
|
1250
|
-
"model": dict(
|
|
1251
|
-
deepcopy(self.stac_config["landing_page"]),
|
|
1252
|
-
**{"provider": self.provider},
|
|
1253
|
-
)
|
|
1254
|
-
}
|
|
1255
|
-
)
|
|
1256
|
-
|
|
1257
|
-
# build children : product_types
|
|
1258
|
-
product_types_list = [
|
|
1259
|
-
pt
|
|
1260
|
-
for pt in self.eodag_api.list_product_types(
|
|
1261
|
-
provider=self.provider, fetch_providers=fetch_providers
|
|
1262
|
-
)
|
|
1263
|
-
]
|
|
1264
|
-
self.set_children(
|
|
1265
|
-
[
|
|
1266
|
-
{
|
|
1267
|
-
"rel": "child",
|
|
1268
|
-
"href": urljoin(
|
|
1269
|
-
self.url, f"{STAC_CATALOGS_PREFIX}/{product_type['ID']}"
|
|
1270
|
-
),
|
|
1271
|
-
"title": product_type["ID"],
|
|
1272
|
-
}
|
|
1273
|
-
for product_type in product_types_list
|
|
1274
|
-
]
|
|
1275
|
-
)
|
|
1276
|
-
else:
|
|
1277
|
-
# use product_types_list as base for building nested catalogs
|
|
1278
|
-
self.__update_data_from_catalog_config(
|
|
1279
|
-
deepcopy(self.stac_config["catalogs"]["product_types_list"])
|
|
1280
|
-
)
|
|
1281
|
-
|
|
1282
|
-
for idx, cat in enumerate(catalogs):
|
|
1283
|
-
if idx % 2 == 0:
|
|
1284
|
-
# even: cat is a filtering value ----------------------------------
|
|
1285
|
-
cat_data_name = self.catalog_config["child_key"]
|
|
1286
|
-
cat_data_value = cat
|
|
1287
|
-
|
|
1288
|
-
# update data
|
|
1289
|
-
set_data_method_name = (
|
|
1290
|
-
"set_stac_%s_by_id" % cat_data_name
|
|
1291
|
-
if "catalog_type"
|
|
1292
|
-
not in self.stac_config["catalogs"][cat_data_name].keys()
|
|
1293
|
-
else "set_stac_%s_by_id"
|
|
1294
|
-
% self.stac_config["catalogs"][cat_data_name]["catalog_type"]
|
|
1295
|
-
)
|
|
1296
|
-
set_data_method = getattr(self, set_data_method_name)
|
|
1297
|
-
set_data_method(cat_data_value, catalog_name=cat_data_name)
|
|
1298
|
-
|
|
1299
|
-
if idx == len(catalogs) - 1:
|
|
1300
|
-
# build children : remaining filtering keys
|
|
1301
|
-
remaining_catalogs_list = [
|
|
1302
|
-
c
|
|
1303
|
-
for c in self.stac_config["catalogs"].keys()
|
|
1304
|
-
# keep filters not used yet AND
|
|
1305
|
-
if self.stac_config["catalogs"][c]["model"]["id"]
|
|
1306
|
-
not in catalogs
|
|
1307
|
-
and (
|
|
1308
|
-
# filters with no parent_key constraint (no key, or key=None) OR
|
|
1309
|
-
"parent_key" not in self.stac_config["catalogs"][c]
|
|
1310
|
-
or not self.stac_config["catalogs"][c]["parent_key"]
|
|
1311
|
-
# filters matching parent_key constraint
|
|
1312
|
-
or self.stac_config["catalogs"][c]["parent_key"]
|
|
1313
|
-
== cat_data_name
|
|
1314
|
-
)
|
|
1315
|
-
# AND filters that match parent attr constraint (locations)
|
|
1316
|
-
and (
|
|
1317
|
-
"parent" not in self.stac_config["catalogs"][c]
|
|
1318
|
-
or not self.stac_config["catalogs"][c]["parent"]["key"]
|
|
1319
|
-
or (
|
|
1320
|
-
self.stac_config["catalogs"][c]["parent"]["key"]
|
|
1321
|
-
== cat_data_name
|
|
1322
|
-
and self.stac_config["catalogs"][c]["parent"]["attr"]
|
|
1323
|
-
== cat_data_value
|
|
1324
|
-
)
|
|
1325
|
-
)
|
|
1326
|
-
]
|
|
1327
|
-
|
|
1328
|
-
self.set_children(
|
|
1329
|
-
[
|
|
1330
|
-
{
|
|
1331
|
-
"rel": "child",
|
|
1332
|
-
"href": self.url
|
|
1333
|
-
+ "/"
|
|
1334
|
-
+ self.stac_config["catalogs"][c]["model"]["id"],
|
|
1335
|
-
"title": str(
|
|
1336
|
-
self.stac_config["catalogs"][c]["model"]["id"]
|
|
1337
|
-
),
|
|
1338
|
-
}
|
|
1339
|
-
for c in remaining_catalogs_list
|
|
1340
|
-
]
|
|
1341
|
-
+ [
|
|
1342
|
-
{
|
|
1343
|
-
"rel": "items",
|
|
1344
|
-
"href": self.url + "/items",
|
|
1345
|
-
"title": "items",
|
|
1346
|
-
}
|
|
1347
|
-
]
|
|
1348
|
-
)
|
|
1349
|
-
|
|
1350
|
-
else:
|
|
1351
|
-
# odd: cat is a filtering key -------------------------------------
|
|
1352
|
-
try:
|
|
1353
|
-
cat_key = [
|
|
1354
|
-
c
|
|
1355
|
-
for c in self.stac_config["catalogs"].keys()
|
|
1356
|
-
if self.stac_config["catalogs"][c]["model"]["id"] == cat
|
|
1357
|
-
][0]
|
|
1358
|
-
except IndexError:
|
|
1359
|
-
raise ValidationError(
|
|
1360
|
-
"Bad settings for %s in stac_config catalogs" % cat
|
|
1361
|
-
)
|
|
1362
|
-
cat_config = deepcopy(self.stac_config["catalogs"][cat_key])
|
|
1363
|
-
# update data
|
|
1364
|
-
self.__update_data_from_catalog_config(cat_config)
|
|
1365
|
-
|
|
1366
|
-
# get filtering values list
|
|
1367
|
-
get_data_method_name = (
|
|
1368
|
-
"get_stac_%s" % cat_key
|
|
1369
|
-
if "catalog_type"
|
|
1370
|
-
not in self.stac_config["catalogs"][cat_key].keys()
|
|
1371
|
-
else "get_stac_%s"
|
|
1372
|
-
% self.stac_config["catalogs"][cat_key]["catalog_type"]
|
|
1373
|
-
)
|
|
1374
|
-
get_data_method = getattr(self, get_data_method_name)
|
|
1375
|
-
cat_data_list = get_data_method(catalog_name=cat_key)
|
|
1376
|
-
|
|
1377
|
-
if idx == len(catalogs) - 1:
|
|
1378
|
-
# filtering values list as children (do not include items)
|
|
1379
|
-
self.set_children(
|
|
1380
|
-
[
|
|
1381
|
-
{
|
|
1382
|
-
"rel": "child",
|
|
1383
|
-
"href": self.url + "/" + str(filtering_data),
|
|
1384
|
-
"title": str(filtering_data),
|
|
1385
|
-
}
|
|
1386
|
-
for filtering_data in cat_data_list
|
|
1387
|
-
]
|
|
1388
|
-
)
|
|
1389
|
-
|
|
1390
|
-
return self
|
|
1391
|
-
|
|
1392
|
-
def get_stac_catalog(self) -> Dict[str, Any]:
|
|
1393
|
-
"""Get nested STAC catalog as data dict
|
|
1394
|
-
|
|
1395
|
-
:returns: Catalog dictionnary
|
|
1396
|
-
:rtype: dict
|
|
1397
|
-
"""
|
|
1398
|
-
return self.as_dict()
|