eodag 3.9.1__py3-none-any.whl → 4.0.0a1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- eodag/api/core.py +378 -419
- eodag/api/product/__init__.py +3 -3
- eodag/api/product/_product.py +68 -40
- eodag/api/product/drivers/__init__.py +3 -5
- eodag/api/product/drivers/base.py +1 -18
- eodag/api/product/metadata_mapping.py +151 -215
- eodag/api/search_result.py +13 -7
- eodag/cli.py +72 -395
- eodag/config.py +46 -50
- eodag/plugins/apis/base.py +2 -2
- eodag/plugins/apis/ecmwf.py +20 -21
- eodag/plugins/apis/usgs.py +37 -33
- eodag/plugins/authentication/aws_auth.py +36 -1
- eodag/plugins/authentication/base.py +18 -3
- eodag/plugins/authentication/sas_auth.py +15 -0
- eodag/plugins/crunch/filter_date.py +3 -3
- eodag/plugins/crunch/filter_latest_intersect.py +2 -2
- eodag/plugins/crunch/filter_latest_tpl_name.py +1 -1
- eodag/plugins/download/aws.py +45 -41
- eodag/plugins/download/base.py +13 -14
- eodag/plugins/download/http.py +65 -65
- eodag/plugins/manager.py +28 -29
- eodag/plugins/search/__init__.py +3 -4
- eodag/plugins/search/base.py +128 -77
- eodag/plugins/search/build_search_result.py +105 -107
- eodag/plugins/search/cop_marine.py +44 -47
- eodag/plugins/search/csw.py +33 -33
- eodag/plugins/search/qssearch.py +335 -354
- eodag/plugins/search/stac_list_assets.py +1 -1
- eodag/plugins/search/static_stac_search.py +31 -31
- eodag/resources/{product_types.yml → collections.yml} +2353 -2429
- eodag/resources/ext_collections.json +1 -0
- eodag/resources/ext_product_types.json +1 -1
- eodag/resources/providers.yml +2432 -2714
- eodag/resources/stac_provider.yml +46 -90
- eodag/types/queryables.py +55 -91
- eodag/types/search_args.py +1 -1
- eodag/utils/__init__.py +94 -21
- eodag/utils/exceptions.py +6 -6
- eodag/utils/free_text_search.py +3 -3
- {eodag-3.9.1.dist-info → eodag-4.0.0a1.dist-info}/METADATA +11 -88
- eodag-4.0.0a1.dist-info/RECORD +92 -0
- {eodag-3.9.1.dist-info → eodag-4.0.0a1.dist-info}/entry_points.txt +0 -4
- eodag/plugins/authentication/oauth.py +0 -60
- eodag/plugins/download/creodias_s3.py +0 -64
- eodag/plugins/download/s3rest.py +0 -351
- eodag/plugins/search/data_request_search.py +0 -565
- eodag/resources/stac.yml +0 -294
- eodag/resources/stac_api.yml +0 -2105
- eodag/rest/__init__.py +0 -24
- eodag/rest/cache.py +0 -70
- eodag/rest/config.py +0 -67
- eodag/rest/constants.py +0 -26
- eodag/rest/core.py +0 -764
- eodag/rest/errors.py +0 -210
- eodag/rest/server.py +0 -604
- eodag/rest/server.wsgi +0 -6
- eodag/rest/stac.py +0 -1032
- eodag/rest/templates/README +0 -1
- eodag/rest/types/__init__.py +0 -18
- eodag/rest/types/collections_search.py +0 -44
- eodag/rest/types/eodag_search.py +0 -386
- eodag/rest/types/queryables.py +0 -174
- eodag/rest/types/stac_search.py +0 -272
- eodag/rest/utils/__init__.py +0 -207
- eodag/rest/utils/cql_evaluate.py +0 -119
- eodag/rest/utils/rfc3339.py +0 -64
- eodag-3.9.1.dist-info/RECORD +0 -115
- {eodag-3.9.1.dist-info → eodag-4.0.0a1.dist-info}/WHEEL +0 -0
- {eodag-3.9.1.dist-info → eodag-4.0.0a1.dist-info}/licenses/LICENSE +0 -0
- {eodag-3.9.1.dist-info → eodag-4.0.0a1.dist-info}/top_level.txt +0 -0
eodag/rest/stac.py
DELETED
|
@@ -1,1032 +0,0 @@
|
|
|
1
|
-
# -*- coding: utf-8 -*-
|
|
2
|
-
# Copyright 2018, CS Systemes d'Information, https://www.csgroup.eu/
|
|
3
|
-
#
|
|
4
|
-
# This file is part of EODAG project
|
|
5
|
-
# https://www.github.com/CS-SI/EODAG
|
|
6
|
-
#
|
|
7
|
-
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
8
|
-
# you may not use this file except in compliance with the License.
|
|
9
|
-
# You may obtain a copy of the License at
|
|
10
|
-
#
|
|
11
|
-
# http://www.apache.org/licenses/LICENSE-2.0
|
|
12
|
-
#
|
|
13
|
-
# Unless required by applicable law or agreed to in writing, software
|
|
14
|
-
# distributed under the License is distributed on an "AS IS" BASIS,
|
|
15
|
-
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
16
|
-
# See the License for the specific language governing permissions and
|
|
17
|
-
# limitations under the License.
|
|
18
|
-
from __future__ import annotations
|
|
19
|
-
|
|
20
|
-
import logging
|
|
21
|
-
import os
|
|
22
|
-
import warnings
|
|
23
|
-
from collections import defaultdict
|
|
24
|
-
from datetime import datetime, timezone
|
|
25
|
-
from typing import TYPE_CHECKING, Any, Optional
|
|
26
|
-
from urllib.parse import (
|
|
27
|
-
parse_qs,
|
|
28
|
-
quote,
|
|
29
|
-
urlencode,
|
|
30
|
-
urlparse,
|
|
31
|
-
urlsplit,
|
|
32
|
-
urlunparse,
|
|
33
|
-
urlunsplit,
|
|
34
|
-
)
|
|
35
|
-
|
|
36
|
-
import geojson
|
|
37
|
-
from jsonpath_ng.jsonpath import Child
|
|
38
|
-
from shapely.geometry import Polygon
|
|
39
|
-
|
|
40
|
-
from eodag.api.product.metadata_mapping import (
|
|
41
|
-
DEFAULT_METADATA_MAPPING,
|
|
42
|
-
format_metadata,
|
|
43
|
-
get_metadata_path,
|
|
44
|
-
)
|
|
45
|
-
from eodag.rest.config import Settings
|
|
46
|
-
from eodag.rest.types.stac_search import SearchPostRequest
|
|
47
|
-
from eodag.rest.utils.rfc3339 import str_to_interval
|
|
48
|
-
from eodag.utils import (
|
|
49
|
-
deepcopy,
|
|
50
|
-
dict_items_recursive_apply,
|
|
51
|
-
format_dict_items,
|
|
52
|
-
guess_file_type,
|
|
53
|
-
jsonpath_parse_dict_items,
|
|
54
|
-
string_to_jsonpath,
|
|
55
|
-
update_nested_dict,
|
|
56
|
-
)
|
|
57
|
-
from eodag.utils.exceptions import (
|
|
58
|
-
NoMatchingProductType,
|
|
59
|
-
NotAvailableError,
|
|
60
|
-
RequestError,
|
|
61
|
-
TimeOutError,
|
|
62
|
-
ValidationError,
|
|
63
|
-
)
|
|
64
|
-
from eodag.utils.requests import fetch_json
|
|
65
|
-
|
|
66
|
-
if TYPE_CHECKING:
|
|
67
|
-
from eodag.api.core import EODataAccessGateway
|
|
68
|
-
from eodag.api.product import EOProduct
|
|
69
|
-
from eodag.api.search_result import SearchResult
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
warnings.warn(
|
|
73
|
-
"The module `eodag.rest.stac` is deprecated since v3.9.0 and will be removed in a future version. "
|
|
74
|
-
"The STAC server has moved to https://github.com/CS-SI/stac-fastapi-eodag",
|
|
75
|
-
category=DeprecationWarning,
|
|
76
|
-
stacklevel=2,
|
|
77
|
-
)
|
|
78
|
-
|
|
79
|
-
logger = logging.getLogger("eodag.rest.stac")
|
|
80
|
-
|
|
81
|
-
# fields not to put in item properties
|
|
82
|
-
COLLECTION_PROPERTIES = [
|
|
83
|
-
"abstract",
|
|
84
|
-
"instrument",
|
|
85
|
-
"platform",
|
|
86
|
-
"platformSerialIdentifier",
|
|
87
|
-
"processingLevel",
|
|
88
|
-
"sensorType",
|
|
89
|
-
"md5",
|
|
90
|
-
"license",
|
|
91
|
-
"title",
|
|
92
|
-
"missionStartDate",
|
|
93
|
-
"missionEndDate",
|
|
94
|
-
"keywords",
|
|
95
|
-
"stacCollection",
|
|
96
|
-
"alias",
|
|
97
|
-
"productType",
|
|
98
|
-
]
|
|
99
|
-
IGNORED_ITEM_PROPERTIES = [
|
|
100
|
-
"_id",
|
|
101
|
-
"id",
|
|
102
|
-
"keyword",
|
|
103
|
-
"quicklook",
|
|
104
|
-
"thumbnail",
|
|
105
|
-
"downloadLink",
|
|
106
|
-
"orderLink",
|
|
107
|
-
"_dc_qs",
|
|
108
|
-
"qs",
|
|
109
|
-
"defaultGeometry",
|
|
110
|
-
"_date",
|
|
111
|
-
"productType",
|
|
112
|
-
]
|
|
113
|
-
|
|
114
|
-
|
|
115
|
-
def _quote_url_path(url: str) -> str:
|
|
116
|
-
parsed = urlsplit(url)
|
|
117
|
-
path = quote(parsed.path)
|
|
118
|
-
components = (parsed.scheme, parsed.netloc, path, parsed.query, parsed.fragment)
|
|
119
|
-
return urlunsplit(components)
|
|
120
|
-
|
|
121
|
-
|
|
122
|
-
class StacCommon:
|
|
123
|
-
"""Stac common object
|
|
124
|
-
|
|
125
|
-
:param url: Requested URL
|
|
126
|
-
:param stac_config: STAC configuration from stac.yml conf file
|
|
127
|
-
:param provider: (optional) Chosen provider
|
|
128
|
-
:param eodag_api: EODAG python API instance
|
|
129
|
-
:param root: (optional) API root
|
|
130
|
-
"""
|
|
131
|
-
|
|
132
|
-
def __init__(
|
|
133
|
-
self,
|
|
134
|
-
url: str,
|
|
135
|
-
stac_config: dict[str, Any],
|
|
136
|
-
provider: Optional[str],
|
|
137
|
-
eodag_api: EODataAccessGateway,
|
|
138
|
-
root: str = "/",
|
|
139
|
-
) -> None:
|
|
140
|
-
self.url = url.rstrip("/") if len(url) > 1 else url
|
|
141
|
-
self.stac_config = stac_config
|
|
142
|
-
self.provider = provider
|
|
143
|
-
self.eodag_api = eodag_api
|
|
144
|
-
self.root = root.rstrip("/") if len(root) > 1 else root
|
|
145
|
-
|
|
146
|
-
self.data: dict[str, Any] = {}
|
|
147
|
-
|
|
148
|
-
def update_data(self, data: dict[str, Any]) -> None:
|
|
149
|
-
"""Updates data using given input STAC dict data
|
|
150
|
-
|
|
151
|
-
:param data: Catalog data (parsed STAC dict)
|
|
152
|
-
"""
|
|
153
|
-
self.data.update(data)
|
|
154
|
-
|
|
155
|
-
# bbox: str to float
|
|
156
|
-
if (
|
|
157
|
-
"extent" in self.data.keys()
|
|
158
|
-
and "spatial" in self.data["extent"].keys()
|
|
159
|
-
and "bbox" in self.data["extent"]["spatial"].keys()
|
|
160
|
-
):
|
|
161
|
-
for i, bbox in enumerate(self.data["extent"]["spatial"]["bbox"]):
|
|
162
|
-
self.data["extent"]["spatial"]["bbox"][i] = [float(x) for x in bbox]
|
|
163
|
-
|
|
164
|
-
def apply_method_none(_: str, v: str) -> Optional[str]:
|
|
165
|
-
""" "None" values to None"""
|
|
166
|
-
return None if v == "None" else v
|
|
167
|
-
|
|
168
|
-
self.data = dict_items_recursive_apply(self.data, apply_method_none)
|
|
169
|
-
|
|
170
|
-
def apply_method_ids(k, v):
|
|
171
|
-
"""ids and titles as str"""
|
|
172
|
-
return str(v) if k in ["title", "id"] else v
|
|
173
|
-
|
|
174
|
-
self.data = dict_items_recursive_apply(self.data, apply_method_ids)
|
|
175
|
-
|
|
176
|
-
# empty stac_extensions: "" to []
|
|
177
|
-
if not self.data.get("stac_extensions", True):
|
|
178
|
-
self.data["stac_extensions"] = []
|
|
179
|
-
|
|
180
|
-
@staticmethod
|
|
181
|
-
def get_stac_extension(
|
|
182
|
-
url: str, stac_config: dict[str, Any], extension: str, **kwargs: Any
|
|
183
|
-
) -> dict[str, str]:
|
|
184
|
-
"""Parse STAC extension from config and return as dict
|
|
185
|
-
|
|
186
|
-
:param url: Requested URL
|
|
187
|
-
:param stac_config: STAC configuration from stac.yml conf file
|
|
188
|
-
:param extension: Extension name
|
|
189
|
-
:param kwargs: Additional variables needed for parsing extension
|
|
190
|
-
:returns: STAC extension as dictionary
|
|
191
|
-
"""
|
|
192
|
-
extension_model = deepcopy(stac_config).get("extensions", {}).get(extension, {})
|
|
193
|
-
|
|
194
|
-
# parse f-strings
|
|
195
|
-
format_args = deepcopy(stac_config)
|
|
196
|
-
format_args["extension"] = {
|
|
197
|
-
"url": url,
|
|
198
|
-
"properties": kwargs.get("properties", {}),
|
|
199
|
-
}
|
|
200
|
-
return format_dict_items(extension_model, **format_args)
|
|
201
|
-
|
|
202
|
-
def get_provider_dict(self, provider: str) -> dict[str, Any]:
|
|
203
|
-
"""Generate STAC provider dict"""
|
|
204
|
-
provider_config = next(
|
|
205
|
-
p
|
|
206
|
-
for p in self.eodag_api.providers_config.values()
|
|
207
|
-
if provider in [p.name, getattr(p, "group", None)]
|
|
208
|
-
)
|
|
209
|
-
return {
|
|
210
|
-
"name": getattr(provider_config, "group", provider_config.name),
|
|
211
|
-
"description": getattr(provider_config, "description", None),
|
|
212
|
-
"roles": getattr(provider_config, "roles", None),
|
|
213
|
-
"url": getattr(provider_config, "url", None),
|
|
214
|
-
"priority": getattr(provider_config, "priority", None),
|
|
215
|
-
}
|
|
216
|
-
|
|
217
|
-
|
|
218
|
-
class StacItem(StacCommon):
|
|
219
|
-
"""Stac item object
|
|
220
|
-
|
|
221
|
-
:param url: Requested URL
|
|
222
|
-
:param stac_config: STAC configuration from stac.yml conf file
|
|
223
|
-
:param provider: (optional) Chosen provider
|
|
224
|
-
:param eodag_api: EODAG python API instance
|
|
225
|
-
:param root: (optional) API root
|
|
226
|
-
"""
|
|
227
|
-
|
|
228
|
-
def __init__(
|
|
229
|
-
self,
|
|
230
|
-
url: str,
|
|
231
|
-
stac_config: dict[str, Any],
|
|
232
|
-
provider: Optional[str],
|
|
233
|
-
eodag_api: EODataAccessGateway,
|
|
234
|
-
root: str = "/",
|
|
235
|
-
) -> None:
|
|
236
|
-
super(StacItem, self).__init__(
|
|
237
|
-
url=url,
|
|
238
|
-
stac_config=stac_config,
|
|
239
|
-
provider=provider,
|
|
240
|
-
eodag_api=eodag_api,
|
|
241
|
-
root=root,
|
|
242
|
-
)
|
|
243
|
-
|
|
244
|
-
def __get_item_list(
|
|
245
|
-
self, search_results: SearchResult, catalog: dict[str, Any]
|
|
246
|
-
) -> list[dict[str, Any]]:
|
|
247
|
-
"""Build STAC items list from EODAG search results
|
|
248
|
-
|
|
249
|
-
:param search_results: EODAG search results
|
|
250
|
-
:param catalog: STAC catalog dict used for parsing item metadata
|
|
251
|
-
:returns: STAC item dicts list
|
|
252
|
-
"""
|
|
253
|
-
if len(search_results) <= 0:
|
|
254
|
-
return []
|
|
255
|
-
|
|
256
|
-
item_model = self.__filter_item_model_properties(
|
|
257
|
-
self.stac_config["item"], str(search_results[0].product_type)
|
|
258
|
-
)
|
|
259
|
-
|
|
260
|
-
# check if some items need to be converted
|
|
261
|
-
need_conversion: dict[str, Any] = {}
|
|
262
|
-
for k, v in item_model["properties"].items():
|
|
263
|
-
if isinstance(v, str):
|
|
264
|
-
conversion, item_model["properties"][k] = get_metadata_path(
|
|
265
|
-
item_model["properties"][k]
|
|
266
|
-
)
|
|
267
|
-
if conversion is not None:
|
|
268
|
-
need_conversion[k] = conversion
|
|
269
|
-
# convert str to jsonpath if needed
|
|
270
|
-
item_model["properties"][k] = string_to_jsonpath(
|
|
271
|
-
k, item_model["properties"][k]
|
|
272
|
-
)
|
|
273
|
-
|
|
274
|
-
item_props = [
|
|
275
|
-
p.right.fields[0]
|
|
276
|
-
for p in item_model["properties"].values()
|
|
277
|
-
if isinstance(p, Child)
|
|
278
|
-
]
|
|
279
|
-
ignored_props = COLLECTION_PROPERTIES + item_props + IGNORED_ITEM_PROPERTIES
|
|
280
|
-
|
|
281
|
-
item_list: list[dict[str, Any]] = []
|
|
282
|
-
for product in search_results:
|
|
283
|
-
product_dict = deepcopy(product.__dict__)
|
|
284
|
-
|
|
285
|
-
product_item: dict[str, Any] = jsonpath_parse_dict_items(
|
|
286
|
-
item_model,
|
|
287
|
-
{
|
|
288
|
-
"product": product_dict,
|
|
289
|
-
"providers": [self.get_provider_dict(product.provider)],
|
|
290
|
-
},
|
|
291
|
-
)
|
|
292
|
-
|
|
293
|
-
# add additional item props
|
|
294
|
-
for p in set(product.properties) - set(ignored_props):
|
|
295
|
-
prefix = getattr(
|
|
296
|
-
self.eodag_api.providers_config[product.provider],
|
|
297
|
-
"group",
|
|
298
|
-
product.provider,
|
|
299
|
-
)
|
|
300
|
-
key = p if ":" in p else f"{prefix}:{p}"
|
|
301
|
-
product_item["properties"][key] = product.properties[p]
|
|
302
|
-
|
|
303
|
-
# parse download link
|
|
304
|
-
downloadlink_href = (
|
|
305
|
-
f"{catalog['url']}/items/{product.properties['id']}/download"
|
|
306
|
-
)
|
|
307
|
-
_dc_qs = product.properties.get("_dc_qs")
|
|
308
|
-
url_parts = urlparse(downloadlink_href)
|
|
309
|
-
query_dict = parse_qs(url_parts.query)
|
|
310
|
-
without_arg_url = (
|
|
311
|
-
f"{url_parts.scheme}://{url_parts.netloc}{url_parts.path}"
|
|
312
|
-
if url_parts.scheme
|
|
313
|
-
else f"{url_parts.netloc}{url_parts.path}"
|
|
314
|
-
)
|
|
315
|
-
# add provider to query-args
|
|
316
|
-
p_config = self.eodag_api.providers_config[product.provider]
|
|
317
|
-
query_dict.update(provider=[getattr(p_config, "group", p_config.name)])
|
|
318
|
-
# add datacube query-string to query-args
|
|
319
|
-
if _dc_qs:
|
|
320
|
-
query_dict.update(_dc_qs=[_dc_qs])
|
|
321
|
-
if query_dict:
|
|
322
|
-
downloadlink_href = (
|
|
323
|
-
f"{without_arg_url}?{urlencode(query_dict, doseq=True)}"
|
|
324
|
-
)
|
|
325
|
-
|
|
326
|
-
# generate STAC assets
|
|
327
|
-
product_item["assets"] = self._get_assets(
|
|
328
|
-
product, downloadlink_href, without_arg_url, query_dict, _dc_qs
|
|
329
|
-
)
|
|
330
|
-
|
|
331
|
-
# apply conversion if needed
|
|
332
|
-
for prop_key, prop_val in need_conversion.items():
|
|
333
|
-
conv_func, conv_args = prop_val
|
|
334
|
-
# colon `:` in key breaks format() method, hide it
|
|
335
|
-
formatable_prop_key = prop_key.replace(":", "")
|
|
336
|
-
if conv_args is not None:
|
|
337
|
-
product_item["properties"][prop_key] = format_metadata(
|
|
338
|
-
"{%s#%s(%s)}" % (formatable_prop_key, conv_func, conv_args),
|
|
339
|
-
**{formatable_prop_key: product_item["properties"][prop_key]},
|
|
340
|
-
)
|
|
341
|
-
else:
|
|
342
|
-
product_item["properties"][prop_key] = format_metadata(
|
|
343
|
-
"{%s#%s}" % (formatable_prop_key, conv_func),
|
|
344
|
-
**{formatable_prop_key: product_item["properties"][prop_key]},
|
|
345
|
-
)
|
|
346
|
-
|
|
347
|
-
# parse f-strings
|
|
348
|
-
format_args = deepcopy(self.stac_config)
|
|
349
|
-
format_args["catalog"] = catalog
|
|
350
|
-
format_args["item"] = product_item
|
|
351
|
-
product_item = format_dict_items(product_item, **format_args)
|
|
352
|
-
product_item["bbox"] = [float(i) for i in product_item["bbox"]]
|
|
353
|
-
|
|
354
|
-
# transform shapely geometry to geojson
|
|
355
|
-
product_item["geometry"] = geojson.loads(
|
|
356
|
-
geojson.dumps(product_item["geometry"])
|
|
357
|
-
)
|
|
358
|
-
|
|
359
|
-
# remove empty properties
|
|
360
|
-
product_item = self.__filter_item_properties_values(product_item)
|
|
361
|
-
|
|
362
|
-
# quote invalid characters in links
|
|
363
|
-
for link in product_item["links"]:
|
|
364
|
-
link["href"] = _quote_url_path(link["href"])
|
|
365
|
-
|
|
366
|
-
# update item link with datacube query-string
|
|
367
|
-
if _dc_qs or self.provider:
|
|
368
|
-
url_parts = urlparse(str(product_item["links"][0]["href"]))
|
|
369
|
-
without_arg_url = (
|
|
370
|
-
f"{url_parts.scheme}://{url_parts.netloc}{url_parts.path}"
|
|
371
|
-
if url_parts.scheme
|
|
372
|
-
else f"{url_parts.netloc}{url_parts.path}"
|
|
373
|
-
)
|
|
374
|
-
product_item["links"][0][
|
|
375
|
-
"href"
|
|
376
|
-
] = f"{without_arg_url}?{urlencode(query_dict, doseq=True)}"
|
|
377
|
-
|
|
378
|
-
item_list.append(product_item)
|
|
379
|
-
|
|
380
|
-
return item_list
|
|
381
|
-
|
|
382
|
-
def _get_assets(
|
|
383
|
-
self,
|
|
384
|
-
product: EOProduct,
|
|
385
|
-
downloadlink_href: str,
|
|
386
|
-
without_arg_url: str,
|
|
387
|
-
query_dict: Optional[dict[str, Any]] = None,
|
|
388
|
-
_dc_qs: Optional[str] = None,
|
|
389
|
-
) -> dict[str, Any]:
|
|
390
|
-
assets: dict[str, Any] = {}
|
|
391
|
-
settings = Settings.from_environment()
|
|
392
|
-
|
|
393
|
-
if _dc_qs:
|
|
394
|
-
parsed = urlparse(product.remote_location)
|
|
395
|
-
fragments = parsed.fragment.split("?")
|
|
396
|
-
parsed = parsed._replace(fragment=f"{fragments[0]}?_dc_qs={_dc_qs}")
|
|
397
|
-
origin_href = urlunparse(parsed)
|
|
398
|
-
else:
|
|
399
|
-
origin_href = product.remote_location
|
|
400
|
-
|
|
401
|
-
# update download link with up-to-date query-args
|
|
402
|
-
quoted_href = _quote_url_path(
|
|
403
|
-
downloadlink_href
|
|
404
|
-
) # quote invalid characters in url
|
|
405
|
-
assets["downloadLink"] = {
|
|
406
|
-
"title": "Download link",
|
|
407
|
-
"href": quoted_href,
|
|
408
|
-
"type": "application/zip",
|
|
409
|
-
}
|
|
410
|
-
|
|
411
|
-
if not origin_href.startswith(tuple(settings.origin_url_blacklist)):
|
|
412
|
-
assets["downloadLink"]["alternate"] = {
|
|
413
|
-
"origin": {
|
|
414
|
-
"title": "Origin asset link",
|
|
415
|
-
"href": origin_href,
|
|
416
|
-
}
|
|
417
|
-
}
|
|
418
|
-
|
|
419
|
-
if "storageStatus" in product.properties:
|
|
420
|
-
assets["downloadLink"]["storage:tier"] = product.properties["storageStatus"]
|
|
421
|
-
|
|
422
|
-
# move origin asset urls to alternate links and replace with eodag-server ones
|
|
423
|
-
if product.assets:
|
|
424
|
-
origin_assets = product.assets.as_dict()
|
|
425
|
-
# replace origin asset urls with eodag-server ones
|
|
426
|
-
for asset_key, asset_value in origin_assets.items():
|
|
427
|
-
# use origin asset as default
|
|
428
|
-
assets[asset_key] = asset_value
|
|
429
|
-
# origin assets as alternate link
|
|
430
|
-
if not asset_value["href"].startswith(
|
|
431
|
-
tuple(settings.origin_url_blacklist)
|
|
432
|
-
):
|
|
433
|
-
assets[asset_key]["alternate"] = {
|
|
434
|
-
"origin": {
|
|
435
|
-
"title": "Origin asset link",
|
|
436
|
-
"href": asset_value["href"],
|
|
437
|
-
}
|
|
438
|
-
}
|
|
439
|
-
# use server-mode assets download links
|
|
440
|
-
asset_value["href"] = without_arg_url
|
|
441
|
-
if query_dict:
|
|
442
|
-
assets[asset_key][
|
|
443
|
-
"href"
|
|
444
|
-
] += f"/{asset_key}?{urlencode(query_dict, doseq=True)}"
|
|
445
|
-
else:
|
|
446
|
-
assets[asset_key]["href"] += f"/{asset_key}"
|
|
447
|
-
if asset_type := asset_value.get("type"):
|
|
448
|
-
assets[asset_key]["type"] = asset_type
|
|
449
|
-
if origin := assets[asset_key].get("alternate", {}).get("origin"):
|
|
450
|
-
origin["type"] = asset_type
|
|
451
|
-
asset_value["href"] = _quote_url_path(asset_value["href"])
|
|
452
|
-
|
|
453
|
-
if thumbnail_url := product.properties.get(
|
|
454
|
-
"quicklook", product.properties.get("thumbnail")
|
|
455
|
-
):
|
|
456
|
-
assets["thumbnail"] = {
|
|
457
|
-
"title": "Thumbnail",
|
|
458
|
-
"href": thumbnail_url,
|
|
459
|
-
"role": "thumbnail",
|
|
460
|
-
}
|
|
461
|
-
if mime_type := guess_file_type(thumbnail_url):
|
|
462
|
-
assets["thumbnail"]["type"] = mime_type
|
|
463
|
-
return assets
|
|
464
|
-
|
|
465
|
-
def get_stac_items(
|
|
466
|
-
self,
|
|
467
|
-
search_results: SearchResult,
|
|
468
|
-
total: int,
|
|
469
|
-
catalog: dict[str, Any],
|
|
470
|
-
next_link: Optional[dict[str, Any]],
|
|
471
|
-
) -> dict[str, Any]:
|
|
472
|
-
"""Build STAC items from EODAG search results
|
|
473
|
-
|
|
474
|
-
:param search_results: EODAG search results
|
|
475
|
-
:param catalog: STAC catalog dict used for parsing item metadata
|
|
476
|
-
:returns: Items dictionary
|
|
477
|
-
"""
|
|
478
|
-
items_model = deepcopy(self.stac_config["items"])
|
|
479
|
-
|
|
480
|
-
if "?" in self.url:
|
|
481
|
-
# search endpoint: use page url as self link
|
|
482
|
-
for i, _ in enumerate(items_model["links"]):
|
|
483
|
-
if items_model["links"][i]["rel"] == "self":
|
|
484
|
-
items_model["links"][i]["href"] = catalog["url"]
|
|
485
|
-
|
|
486
|
-
timestamp = datetime.now(timezone.utc).isoformat().replace("+00:00", "Z")
|
|
487
|
-
|
|
488
|
-
# parse jsonpath
|
|
489
|
-
items = jsonpath_parse_dict_items(
|
|
490
|
-
items_model,
|
|
491
|
-
{
|
|
492
|
-
"numberMatched": total,
|
|
493
|
-
"numberReturned": len(search_results),
|
|
494
|
-
"timeStamp": timestamp,
|
|
495
|
-
},
|
|
496
|
-
)
|
|
497
|
-
# parse f-strings
|
|
498
|
-
format_args = deepcopy(self.stac_config)
|
|
499
|
-
format_args["catalog"] = catalog
|
|
500
|
-
items = format_dict_items(items, **format_args)
|
|
501
|
-
|
|
502
|
-
if next_link:
|
|
503
|
-
items["links"].append(next_link)
|
|
504
|
-
|
|
505
|
-
# provide static catalog to build features
|
|
506
|
-
if "search?" in catalog["url"]:
|
|
507
|
-
catalog["url"] = os.path.join(
|
|
508
|
-
catalog["url"].split("search?")[0],
|
|
509
|
-
"collections",
|
|
510
|
-
catalog["id"],
|
|
511
|
-
)
|
|
512
|
-
else:
|
|
513
|
-
catalog["url"] = catalog["url"].split("?")[0]
|
|
514
|
-
items["features"] = self.__get_item_list(search_results, catalog)
|
|
515
|
-
|
|
516
|
-
self.update_data(items)
|
|
517
|
-
return self.data
|
|
518
|
-
|
|
519
|
-
def __filter_item_model_properties(
|
|
520
|
-
self, item_model: dict[str, Any], product_type: str
|
|
521
|
-
) -> dict[str, Any]:
|
|
522
|
-
"""Filter item model depending on product type metadata and its extensions.
|
|
523
|
-
Removes not needed parameters, and adds supplementary ones as
|
|
524
|
-
part of oseo extension.
|
|
525
|
-
|
|
526
|
-
:param item_model: Item model from stac_config
|
|
527
|
-
:param product_type: Product type
|
|
528
|
-
:returns: Filtered item model
|
|
529
|
-
"""
|
|
530
|
-
try:
|
|
531
|
-
product_type_dict = [
|
|
532
|
-
pt
|
|
533
|
-
for pt in self.eodag_api.list_product_types(
|
|
534
|
-
provider=self.provider, fetch_providers=False
|
|
535
|
-
)
|
|
536
|
-
if pt["ID"] == product_type
|
|
537
|
-
or ("alias" in pt and pt["alias"] == product_type)
|
|
538
|
-
][0]
|
|
539
|
-
except IndexError as e:
|
|
540
|
-
raise NoMatchingProductType(
|
|
541
|
-
f"Product type {product_type} not available for {self.provider}"
|
|
542
|
-
) from e
|
|
543
|
-
|
|
544
|
-
result_item_model = deepcopy(item_model)
|
|
545
|
-
result_item_model["stac_extensions"] = list(
|
|
546
|
-
self.stac_config["stac_extensions"].values()
|
|
547
|
-
)
|
|
548
|
-
|
|
549
|
-
# build jsonpath for eodag product default properties and adapt path
|
|
550
|
-
eodag_properties_dict = {
|
|
551
|
-
k: string_to_jsonpath(k, v.replace("$.", "$.product."))
|
|
552
|
-
for k, v in DEFAULT_METADATA_MAPPING.items()
|
|
553
|
-
if "$.properties." in v
|
|
554
|
-
}
|
|
555
|
-
# add missing properties as oseo:missingProperty
|
|
556
|
-
for k, v in eodag_properties_dict.items():
|
|
557
|
-
if (
|
|
558
|
-
v not in result_item_model["properties"].values()
|
|
559
|
-
and k not in self.stac_config["metadata_ignore"]
|
|
560
|
-
and not any(
|
|
561
|
-
k in str(prop) for prop in result_item_model["properties"].values()
|
|
562
|
-
)
|
|
563
|
-
):
|
|
564
|
-
result_item_model["properties"]["oseo:" + k] = string_to_jsonpath(k, v)
|
|
565
|
-
|
|
566
|
-
# Filter out unneeded extensions
|
|
567
|
-
if product_type_dict.get("sensorType", "RADAR") != "RADAR":
|
|
568
|
-
result_item_model["stac_extensions"].remove(
|
|
569
|
-
self.stac_config["stac_extensions"]["sar"]
|
|
570
|
-
)
|
|
571
|
-
|
|
572
|
-
# Filter out unneeded properties
|
|
573
|
-
extensions_prefixes = [
|
|
574
|
-
k
|
|
575
|
-
for k, v in self.stac_config["stac_extensions"].items()
|
|
576
|
-
if v in result_item_model["stac_extensions"]
|
|
577
|
-
]
|
|
578
|
-
for k, v in item_model["properties"].items():
|
|
579
|
-
# remove key if extension not in stac_extensions
|
|
580
|
-
if ":" in k and k.split(":")[0] not in extensions_prefixes:
|
|
581
|
-
result_item_model["properties"].pop(k, None)
|
|
582
|
-
|
|
583
|
-
return result_item_model
|
|
584
|
-
|
|
585
|
-
def __filter_item_properties_values(self, item: dict[str, Any]) -> dict[str, Any]:
|
|
586
|
-
"""Removes empty properties, unused extensions, and add missing extensions
|
|
587
|
-
|
|
588
|
-
:param item: STAC item data
|
|
589
|
-
:returns: Filtered item model
|
|
590
|
-
"""
|
|
591
|
-
all_extensions_dict: dict[str, str] = deepcopy(
|
|
592
|
-
self.stac_config["stac_extensions"]
|
|
593
|
-
)
|
|
594
|
-
# parse f-strings with root
|
|
595
|
-
all_extensions_dict = format_dict_items(
|
|
596
|
-
all_extensions_dict, **{"catalog": {"root": self.root}}
|
|
597
|
-
)
|
|
598
|
-
|
|
599
|
-
item["stac_extensions"] = []
|
|
600
|
-
# dict to list of keys to permit pop() while iterating
|
|
601
|
-
for k in list(item["properties"]):
|
|
602
|
-
extension_prefix: str = k.split(":")[0]
|
|
603
|
-
|
|
604
|
-
if item["properties"][k] is None:
|
|
605
|
-
item["properties"].pop(k, None)
|
|
606
|
-
# feed found extensions list
|
|
607
|
-
elif (
|
|
608
|
-
extension_prefix in all_extensions_dict.keys()
|
|
609
|
-
and all_extensions_dict[extension_prefix] not in item["stac_extensions"]
|
|
610
|
-
):
|
|
611
|
-
# append path from item extensions
|
|
612
|
-
item["stac_extensions"].append(all_extensions_dict[extension_prefix])
|
|
613
|
-
|
|
614
|
-
return item
|
|
615
|
-
|
|
616
|
-
def get_stac_item_from_product(self, product: EOProduct) -> dict[str, Any]:
|
|
617
|
-
"""Build STAC item from EODAG product
|
|
618
|
-
|
|
619
|
-
:param product: EODAG product
|
|
620
|
-
:returns: STAC item
|
|
621
|
-
"""
|
|
622
|
-
product_type = str(product.product_type)
|
|
623
|
-
|
|
624
|
-
item_model = self.__filter_item_model_properties(
|
|
625
|
-
self.stac_config["item"], product_type
|
|
626
|
-
)
|
|
627
|
-
|
|
628
|
-
catalog = StacCatalog(
|
|
629
|
-
url=self.url.split("/items")[0],
|
|
630
|
-
stac_config=self.stac_config,
|
|
631
|
-
root=self.root,
|
|
632
|
-
provider=self.provider,
|
|
633
|
-
eodag_api=self.eodag_api,
|
|
634
|
-
collection=product_type,
|
|
635
|
-
)
|
|
636
|
-
|
|
637
|
-
product_dict = deepcopy(product.__dict__)
|
|
638
|
-
product_dict["assets"] = product.assets.as_dict()
|
|
639
|
-
|
|
640
|
-
# parse jsonpath
|
|
641
|
-
product_item = jsonpath_parse_dict_items(
|
|
642
|
-
item_model,
|
|
643
|
-
{
|
|
644
|
-
"product": product_dict,
|
|
645
|
-
"providers": [self.get_provider_dict(product.provider)],
|
|
646
|
-
},
|
|
647
|
-
)
|
|
648
|
-
# parse f-strings
|
|
649
|
-
format_args = deepcopy(self.stac_config)
|
|
650
|
-
format_args["catalog"] = {
|
|
651
|
-
**catalog.data,
|
|
652
|
-
**{"url": catalog.url, "root": catalog.root},
|
|
653
|
-
}
|
|
654
|
-
format_args["item"] = product_item
|
|
655
|
-
product_item = format_dict_items(product_item, **format_args)
|
|
656
|
-
product_item["bbox"] = [float(i) for i in product_item["bbox"]]
|
|
657
|
-
|
|
658
|
-
# remove empty properties
|
|
659
|
-
product_item = self.__filter_item_properties_values(product_item)
|
|
660
|
-
|
|
661
|
-
self.update_data(product_item)
|
|
662
|
-
return self.data
|
|
663
|
-
|
|
664
|
-
|
|
665
|
-
class StacCollection(StacCommon):
|
|
666
|
-
"""Stac collection object
|
|
667
|
-
|
|
668
|
-
:param url: Requested URL
|
|
669
|
-
:param stac_config: STAC configuration from stac.yml conf file
|
|
670
|
-
:param provider: (optional) Chosen provider
|
|
671
|
-
:param eodag_api: EODAG python API instance
|
|
672
|
-
:param root: (optional) API root
|
|
673
|
-
"""
|
|
674
|
-
|
|
675
|
-
# External STAC collections
|
|
676
|
-
ext_stac_collections: dict[str, dict[str, Any]] = dict()
|
|
677
|
-
|
|
678
|
-
@classmethod
|
|
679
|
-
def fetch_external_stac_collections(cls, eodag_api: EODataAccessGateway) -> None:
|
|
680
|
-
"""Load external STAC collections
|
|
681
|
-
|
|
682
|
-
:param eodag_api: EODAG python API instance
|
|
683
|
-
"""
|
|
684
|
-
list_product_types = eodag_api.list_product_types(fetch_providers=False)
|
|
685
|
-
for product_type in list_product_types:
|
|
686
|
-
ext_stac_collection_path = product_type.get("stacCollection")
|
|
687
|
-
if not ext_stac_collection_path:
|
|
688
|
-
continue
|
|
689
|
-
logger.info(f"Fetching external STAC collection for {product_type['ID']}")
|
|
690
|
-
|
|
691
|
-
try:
|
|
692
|
-
ext_stac_collection = fetch_json(ext_stac_collection_path)
|
|
693
|
-
except (RequestError, TimeOutError) as e:
|
|
694
|
-
logger.debug(e)
|
|
695
|
-
logger.warning(
|
|
696
|
-
f"Could not read remote external STAC collection from {ext_stac_collection_path}",
|
|
697
|
-
)
|
|
698
|
-
ext_stac_collection = {}
|
|
699
|
-
|
|
700
|
-
cls.ext_stac_collections[product_type["ID"]] = ext_stac_collection
|
|
701
|
-
|
|
702
|
-
def __init__(
|
|
703
|
-
self,
|
|
704
|
-
url: str,
|
|
705
|
-
stac_config: dict[str, Any],
|
|
706
|
-
provider: Optional[str],
|
|
707
|
-
eodag_api: EODataAccessGateway,
|
|
708
|
-
root: str = "/",
|
|
709
|
-
) -> None:
|
|
710
|
-
super(StacCollection, self).__init__(
|
|
711
|
-
url=url,
|
|
712
|
-
stac_config=stac_config,
|
|
713
|
-
provider=provider,
|
|
714
|
-
eodag_api=eodag_api,
|
|
715
|
-
root=root,
|
|
716
|
-
)
|
|
717
|
-
|
|
718
|
-
def __list_product_type_providers(self, product_type: dict[str, Any]) -> list[str]:
|
|
719
|
-
"""Retrieve a list of providers for a given product type.
|
|
720
|
-
|
|
721
|
-
:param product_type: Dictionary containing information about the product type.
|
|
722
|
-
:return: A list of provider names.
|
|
723
|
-
"""
|
|
724
|
-
if self.provider:
|
|
725
|
-
return [self.provider]
|
|
726
|
-
|
|
727
|
-
return [
|
|
728
|
-
plugin.provider
|
|
729
|
-
for plugin in self.eodag_api._plugins_manager.get_search_plugins(
|
|
730
|
-
product_type=product_type.get("_id", product_type["ID"])
|
|
731
|
-
)
|
|
732
|
-
]
|
|
733
|
-
|
|
734
|
-
def __generate_stac_collection(
|
|
735
|
-
self, collection_model: Any, product_type: dict[str, Any]
|
|
736
|
-
) -> dict[str, Any]:
|
|
737
|
-
"""Generate a STAC collection dictionary for a given product type.
|
|
738
|
-
|
|
739
|
-
:param collection_model: The base model for the STAC collection.
|
|
740
|
-
:param product_type: Dictionary containing information about the product type.
|
|
741
|
-
:return: A dictionary representing the STAC collection for the given product type.
|
|
742
|
-
"""
|
|
743
|
-
providers = self.__list_product_type_providers(product_type)
|
|
744
|
-
|
|
745
|
-
providers_dict: dict[str, dict[str, Any]] = {}
|
|
746
|
-
for provider in providers:
|
|
747
|
-
p_dict = self.get_provider_dict(provider)
|
|
748
|
-
providers_dict.setdefault(p_dict["name"], p_dict)
|
|
749
|
-
providers_list = list(providers_dict.values())
|
|
750
|
-
|
|
751
|
-
# parse jsonpath
|
|
752
|
-
product_type_collection = jsonpath_parse_dict_items(
|
|
753
|
-
collection_model,
|
|
754
|
-
{
|
|
755
|
-
"product_type": product_type,
|
|
756
|
-
"providers": providers_list,
|
|
757
|
-
},
|
|
758
|
-
)
|
|
759
|
-
# override EODAG's collection with the external collection
|
|
760
|
-
ext_stac_collection = deepcopy(
|
|
761
|
-
self.ext_stac_collections.get(product_type["ID"], {})
|
|
762
|
-
)
|
|
763
|
-
|
|
764
|
-
# update links (keep eodag links as defaults)
|
|
765
|
-
ext_stac_collection.setdefault("links", {})
|
|
766
|
-
for link in product_type_collection["links"]:
|
|
767
|
-
ext_stac_collection["links"] = [
|
|
768
|
-
x for x in ext_stac_collection["links"] if x["rel"] != link["rel"]
|
|
769
|
-
]
|
|
770
|
-
ext_stac_collection["links"].append(link)
|
|
771
|
-
|
|
772
|
-
# merge "summaries"
|
|
773
|
-
ext_stac_collection["summaries"] = {
|
|
774
|
-
k: v
|
|
775
|
-
for k, v in {
|
|
776
|
-
**ext_stac_collection.get("summaries", {}),
|
|
777
|
-
**product_type_collection["summaries"],
|
|
778
|
-
}.items()
|
|
779
|
-
if v and any(v)
|
|
780
|
-
}
|
|
781
|
-
|
|
782
|
-
# merge "keywords" lists
|
|
783
|
-
try:
|
|
784
|
-
ext_stac_collection["keywords"] = [
|
|
785
|
-
k
|
|
786
|
-
for k in set(
|
|
787
|
-
ext_stac_collection.get("keywords", [])
|
|
788
|
-
+ product_type_collection["keywords"]
|
|
789
|
-
)
|
|
790
|
-
if k is not None
|
|
791
|
-
]
|
|
792
|
-
except TypeError as e:
|
|
793
|
-
logger.warning(
|
|
794
|
-
f"Could not merge keywords from external collection for {product_type['ID']}: {str(e)}"
|
|
795
|
-
)
|
|
796
|
-
logger.debug(
|
|
797
|
-
f"External collection keywords: {str(ext_stac_collection.get('keywords'))}, ",
|
|
798
|
-
f"Product type keywords: {str(product_type_collection['keywords'])}",
|
|
799
|
-
)
|
|
800
|
-
|
|
801
|
-
product_type_collection.update(ext_stac_collection)
|
|
802
|
-
|
|
803
|
-
# parse f-strings
|
|
804
|
-
format_args = deepcopy(self.stac_config)
|
|
805
|
-
format_args["collection"] = {
|
|
806
|
-
**product_type_collection,
|
|
807
|
-
**{
|
|
808
|
-
"url": self.url
|
|
809
|
-
if self.url.endswith(product_type["ID"])
|
|
810
|
-
else f"{self.url}/{product_type['ID']}",
|
|
811
|
-
"root": self.root,
|
|
812
|
-
},
|
|
813
|
-
}
|
|
814
|
-
product_type_collection = format_dict_items(
|
|
815
|
-
product_type_collection, **format_args
|
|
816
|
-
)
|
|
817
|
-
|
|
818
|
-
return product_type_collection
|
|
819
|
-
|
|
820
|
-
def get_collection_list(
|
|
821
|
-
self,
|
|
822
|
-
collection: Optional[str] = None,
|
|
823
|
-
q: Optional[str] = None,
|
|
824
|
-
platform: Optional[str] = None,
|
|
825
|
-
instrument: Optional[str] = None,
|
|
826
|
-
constellation: Optional[str] = None,
|
|
827
|
-
datetime: Optional[str] = None,
|
|
828
|
-
bbox: Optional[str] = None,
|
|
829
|
-
) -> list[dict[str, Any]]:
|
|
830
|
-
"""Build STAC collections list
|
|
831
|
-
|
|
832
|
-
:param filters: (optional) Additional filters for collections search
|
|
833
|
-
:returns: STAC collection dicts list
|
|
834
|
-
"""
|
|
835
|
-
collection_model = deepcopy(self.stac_config["collection"])
|
|
836
|
-
|
|
837
|
-
start, end = str_to_interval(datetime)
|
|
838
|
-
|
|
839
|
-
all_pt = self.eodag_api.list_product_types(
|
|
840
|
-
provider=self.provider, fetch_providers=False
|
|
841
|
-
)
|
|
842
|
-
|
|
843
|
-
if any((collection, q, platform, instrument, constellation, datetime)):
|
|
844
|
-
try:
|
|
845
|
-
guessed_product_types = self.eodag_api.guess_product_type(
|
|
846
|
-
free_text=q,
|
|
847
|
-
platformSerialIdentifier=platform,
|
|
848
|
-
instrument=instrument,
|
|
849
|
-
platform=constellation,
|
|
850
|
-
productType=collection,
|
|
851
|
-
missionStartDate=start.isoformat() if start else None,
|
|
852
|
-
missionEndDate=end.isoformat() if end else None,
|
|
853
|
-
)
|
|
854
|
-
except NoMatchingProductType:
|
|
855
|
-
product_types = []
|
|
856
|
-
else:
|
|
857
|
-
product_types = [
|
|
858
|
-
pt for pt in all_pt if pt["ID"] in guessed_product_types
|
|
859
|
-
]
|
|
860
|
-
else:
|
|
861
|
-
product_types = all_pt
|
|
862
|
-
|
|
863
|
-
_bbox_poly = None
|
|
864
|
-
if bbox:
|
|
865
|
-
try:
|
|
866
|
-
_bbox = [float(x) for x in bbox.split(",")]
|
|
867
|
-
SearchPostRequest.validate_bbox(_bbox) # type: ignore
|
|
868
|
-
_bbox_poly = Polygon.from_bounds(*_bbox)
|
|
869
|
-
except ValueError as e:
|
|
870
|
-
raise ValidationError(f"Wrong bbox: {e}")
|
|
871
|
-
|
|
872
|
-
# list product types with all metadata using guessed ids
|
|
873
|
-
collection_list: list[dict[str, Any]] = []
|
|
874
|
-
for product_type in product_types:
|
|
875
|
-
stac_collection = self.__generate_stac_collection(
|
|
876
|
-
collection_model, product_type
|
|
877
|
-
)
|
|
878
|
-
# Apply bbox filter
|
|
879
|
-
if _bbox_poly:
|
|
880
|
-
_other_bbox_poly = Polygon.from_bounds(
|
|
881
|
-
*stac_collection["extent"]["spatial"]["bbox"][0]
|
|
882
|
-
)
|
|
883
|
-
if _bbox_poly.intersects(_other_bbox_poly):
|
|
884
|
-
collection_list.append(stac_collection)
|
|
885
|
-
else:
|
|
886
|
-
collection_list.append(stac_collection)
|
|
887
|
-
|
|
888
|
-
return collection_list
|
|
889
|
-
|
|
890
|
-
|
|
891
|
-
class StacCatalog(StacCommon):
|
|
892
|
-
"""Stac Catalog object
|
|
893
|
-
|
|
894
|
-
:param url: Requested URL
|
|
895
|
-
:param stac_config: STAC configuration from stac.yml conf file
|
|
896
|
-
:param provider: Chosen provider
|
|
897
|
-
:param eodag_api: EODAG python API instance
|
|
898
|
-
:param root: (optional) API root
|
|
899
|
-
:param collection: (optional) product type id
|
|
900
|
-
"""
|
|
901
|
-
|
|
902
|
-
def __init__(
|
|
903
|
-
self,
|
|
904
|
-
url: str,
|
|
905
|
-
stac_config: dict[str, Any],
|
|
906
|
-
provider: Optional[str],
|
|
907
|
-
eodag_api: EODataAccessGateway,
|
|
908
|
-
root: str = "/",
|
|
909
|
-
collection: Optional[str] = None,
|
|
910
|
-
) -> None:
|
|
911
|
-
super(StacCatalog, self).__init__(
|
|
912
|
-
url=url,
|
|
913
|
-
stac_config=stac_config,
|
|
914
|
-
provider=provider,
|
|
915
|
-
eodag_api=eodag_api,
|
|
916
|
-
root=root,
|
|
917
|
-
)
|
|
918
|
-
self.data = {}
|
|
919
|
-
|
|
920
|
-
self.shp_location_config = eodag_api.locations_config
|
|
921
|
-
self.search_args: dict[str, Any] = {}
|
|
922
|
-
self.children: list[dict[str, Any]] = []
|
|
923
|
-
|
|
924
|
-
self.catalog_config = deepcopy(stac_config["catalog"])
|
|
925
|
-
|
|
926
|
-
self.__update_data_from_catalog_config({"model": {}})
|
|
927
|
-
|
|
928
|
-
# expand links
|
|
929
|
-
self.parent = "/".join(self.url.rstrip("/").split("/")[:-1])
|
|
930
|
-
if self.parent != self.root:
|
|
931
|
-
self.data["links"].append({"rel": "parent", "href": self.parent})
|
|
932
|
-
if self.children:
|
|
933
|
-
self.data["links"] += self.children
|
|
934
|
-
|
|
935
|
-
# build catalog
|
|
936
|
-
self.__build_stac_catalog(collection)
|
|
937
|
-
|
|
938
|
-
def __update_data_from_catalog_config(self, catalog_config: dict[str, Any]) -> bool:
|
|
939
|
-
"""Updates configuration and data using given input catalog config
|
|
940
|
-
|
|
941
|
-
:param catalog_config: Catalog config, from yml stac_config[catalogs]
|
|
942
|
-
"""
|
|
943
|
-
model = catalog_config["model"]
|
|
944
|
-
|
|
945
|
-
self.catalog_config = update_nested_dict(self.catalog_config, catalog_config)
|
|
946
|
-
|
|
947
|
-
# parse f-strings
|
|
948
|
-
# defaultdict usage will return "" for missing keys in format_args
|
|
949
|
-
format_args = deepcopy(self.stac_config)
|
|
950
|
-
format_args["catalog"] = defaultdict(
|
|
951
|
-
str, dict(model, **{"root": self.root, "url": self.url})
|
|
952
|
-
)
|
|
953
|
-
# use existing data as parent_catalog
|
|
954
|
-
format_args["parent_catalog"] = defaultdict(str, **self.data)
|
|
955
|
-
parsed_model = format_dict_items(self.catalog_config["model"], **format_args)
|
|
956
|
-
|
|
957
|
-
self.update_data(parsed_model)
|
|
958
|
-
|
|
959
|
-
return True
|
|
960
|
-
|
|
961
|
-
def __build_stac_catalog(self, collection: Optional[str] = None) -> StacCatalog:
|
|
962
|
-
"""Build nested catalog from catalag list
|
|
963
|
-
|
|
964
|
-
:param collection: (optional) product type id
|
|
965
|
-
:returns: This catalog obj
|
|
966
|
-
"""
|
|
967
|
-
settings = Settings.from_environment()
|
|
968
|
-
|
|
969
|
-
if not collection:
|
|
970
|
-
# Build root catalog combined with landing page
|
|
971
|
-
self.__update_data_from_catalog_config(
|
|
972
|
-
{
|
|
973
|
-
"model": {
|
|
974
|
-
**deepcopy(self.stac_config["landing_page"]),
|
|
975
|
-
**{
|
|
976
|
-
"provider": self.provider,
|
|
977
|
-
"id": settings.stac_api_landing_id,
|
|
978
|
-
"title": settings.stac_api_title,
|
|
979
|
-
"description": settings.stac_api_description,
|
|
980
|
-
},
|
|
981
|
-
}
|
|
982
|
-
}
|
|
983
|
-
)
|
|
984
|
-
else:
|
|
985
|
-
self.set_stac_product_type_by_id(collection)
|
|
986
|
-
return self
|
|
987
|
-
|
|
988
|
-
def set_stac_product_type_by_id(
|
|
989
|
-
self, product_type: str, **_: Any
|
|
990
|
-
) -> dict[str, Any]:
|
|
991
|
-
"""Updates catalog with given product_type
|
|
992
|
-
|
|
993
|
-
:param product_type: Product type
|
|
994
|
-
"""
|
|
995
|
-
collections = StacCollection(
|
|
996
|
-
url=self.url,
|
|
997
|
-
stac_config=self.stac_config,
|
|
998
|
-
provider=self.provider,
|
|
999
|
-
eodag_api=self.eodag_api,
|
|
1000
|
-
root=self.root,
|
|
1001
|
-
).get_collection_list(collection=product_type)
|
|
1002
|
-
|
|
1003
|
-
if not collections:
|
|
1004
|
-
raise NotAvailableError(f"Collection {product_type} does not exist.")
|
|
1005
|
-
|
|
1006
|
-
cat_model = {
|
|
1007
|
-
"id": "{collection[id]}",
|
|
1008
|
-
"title": "{collection[title]}",
|
|
1009
|
-
"description": "{collection[description]}",
|
|
1010
|
-
"extent": "{collection[extent]}",
|
|
1011
|
-
"crs": "http://www.opengis.net/def/crs/OGC/1.3/CRS84",
|
|
1012
|
-
"keywords": "{collection[keywords]}",
|
|
1013
|
-
"license": "{collection[license]}",
|
|
1014
|
-
"providers": "{collection[providers]}",
|
|
1015
|
-
"summaries": "{collection[summaries]}",
|
|
1016
|
-
}
|
|
1017
|
-
# parse f-strings
|
|
1018
|
-
format_args = deepcopy(self.stac_config)
|
|
1019
|
-
format_args["catalog"] = defaultdict(str, **self.data)
|
|
1020
|
-
format_args["collection"] = collections[0]
|
|
1021
|
-
try:
|
|
1022
|
-
parsed_dict: dict[str, Any] = format_dict_items(cat_model, **format_args)
|
|
1023
|
-
except Exception:
|
|
1024
|
-
logger.error("Could not format product_type catalog")
|
|
1025
|
-
raise
|
|
1026
|
-
|
|
1027
|
-
self.update_data(parsed_dict)
|
|
1028
|
-
|
|
1029
|
-
# update search args
|
|
1030
|
-
self.search_args.update({"productType": product_type})
|
|
1031
|
-
|
|
1032
|
-
return parsed_dict
|