eodag 2.12.1__py3-none-any.whl → 3.0.0b1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- eodag/api/core.py +434 -319
- eodag/api/product/__init__.py +5 -1
- eodag/api/product/_assets.py +7 -2
- eodag/api/product/_product.py +46 -68
- eodag/api/product/metadata_mapping.py +181 -66
- eodag/api/search_result.py +21 -1
- eodag/cli.py +20 -6
- eodag/config.py +95 -6
- eodag/plugins/apis/base.py +8 -165
- eodag/plugins/apis/ecmwf.py +36 -24
- eodag/plugins/apis/usgs.py +40 -24
- eodag/plugins/authentication/aws_auth.py +2 -2
- eodag/plugins/authentication/header.py +31 -6
- eodag/plugins/authentication/keycloak.py +13 -84
- eodag/plugins/authentication/oauth.py +3 -3
- eodag/plugins/authentication/openid_connect.py +256 -46
- eodag/plugins/authentication/qsauth.py +3 -0
- eodag/plugins/authentication/sas_auth.py +8 -1
- eodag/plugins/authentication/token.py +92 -46
- eodag/plugins/authentication/token_exchange.py +120 -0
- eodag/plugins/download/aws.py +86 -91
- eodag/plugins/download/base.py +72 -40
- eodag/plugins/download/http.py +607 -264
- eodag/plugins/download/s3rest.py +28 -15
- eodag/plugins/manager.py +73 -57
- eodag/plugins/search/__init__.py +36 -0
- eodag/plugins/search/base.py +225 -18
- eodag/plugins/search/build_search_result.py +389 -32
- eodag/plugins/search/cop_marine.py +378 -0
- eodag/plugins/search/creodias_s3.py +15 -14
- eodag/plugins/search/csw.py +5 -7
- eodag/plugins/search/data_request_search.py +44 -20
- eodag/plugins/search/qssearch.py +508 -203
- eodag/plugins/search/static_stac_search.py +99 -36
- eodag/resources/constraints/climate-dt.json +13 -0
- eodag/resources/constraints/extremes-dt.json +8 -0
- eodag/resources/ext_product_types.json +1 -1
- eodag/resources/product_types.yml +1897 -34
- eodag/resources/providers.yml +3539 -3277
- eodag/resources/stac.yml +48 -54
- eodag/resources/stac_api.yml +71 -25
- eodag/resources/stac_provider.yml +5 -0
- eodag/resources/user_conf_template.yml +51 -3
- eodag/rest/__init__.py +6 -0
- eodag/rest/cache.py +70 -0
- eodag/rest/config.py +68 -0
- eodag/rest/constants.py +27 -0
- eodag/rest/core.py +757 -0
- eodag/rest/server.py +397 -258
- eodag/rest/stac.py +438 -307
- eodag/rest/types/collections_search.py +44 -0
- eodag/rest/types/eodag_search.py +232 -43
- eodag/rest/types/{stac_queryables.py → queryables.py} +81 -43
- eodag/rest/types/stac_search.py +277 -0
- eodag/rest/utils/__init__.py +216 -0
- eodag/rest/utils/cql_evaluate.py +119 -0
- eodag/rest/utils/rfc3339.py +65 -0
- eodag/types/__init__.py +99 -9
- eodag/types/bbox.py +15 -14
- eodag/types/download_args.py +31 -0
- eodag/types/search_args.py +58 -7
- eodag/types/whoosh.py +81 -0
- eodag/utils/__init__.py +72 -9
- eodag/utils/constraints.py +37 -37
- eodag/utils/exceptions.py +23 -17
- eodag/utils/requests.py +138 -0
- eodag/utils/rest.py +104 -0
- eodag/utils/stac_reader.py +100 -16
- {eodag-2.12.1.dist-info → eodag-3.0.0b1.dist-info}/METADATA +64 -44
- eodag-3.0.0b1.dist-info/RECORD +109 -0
- {eodag-2.12.1.dist-info → eodag-3.0.0b1.dist-info}/WHEEL +1 -1
- {eodag-2.12.1.dist-info → eodag-3.0.0b1.dist-info}/entry_points.txt +6 -5
- eodag/plugins/apis/cds.py +0 -540
- eodag/rest/utils.py +0 -1133
- eodag-2.12.1.dist-info/RECORD +0 -94
- {eodag-2.12.1.dist-info → eodag-3.0.0b1.dist-info}/LICENSE +0 -0
- {eodag-2.12.1.dist-info → eodag-3.0.0b1.dist-info}/top_level.txt +0 -0
eodag/rest/core.py
ADDED
|
@@ -0,0 +1,757 @@
|
|
|
1
|
+
# -*- coding: utf-8 -*-
|
|
2
|
+
# Copyright 2023, CS GROUP - France, https://www.csgroup.eu/
|
|
3
|
+
#
|
|
4
|
+
# This file is part of EODAG project
|
|
5
|
+
# https://www.github.com/CS-SI/EODAG
|
|
6
|
+
#
|
|
7
|
+
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
8
|
+
# you may not use this file except in compliance with the License.
|
|
9
|
+
# You may obtain a copy of the License at
|
|
10
|
+
#
|
|
11
|
+
# http://www.apache.org/licenses/LICENSE-2.0
|
|
12
|
+
#
|
|
13
|
+
# Unless required by applicable law or agreed to in writing, software
|
|
14
|
+
# distributed under the License is distributed on an "AS IS" BASIS,
|
|
15
|
+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
16
|
+
# See the License for the specific language governing permissions and
|
|
17
|
+
# limitations under the License.
|
|
18
|
+
from __future__ import annotations
|
|
19
|
+
|
|
20
|
+
import datetime
|
|
21
|
+
import logging
|
|
22
|
+
import os
|
|
23
|
+
import re
|
|
24
|
+
from typing import TYPE_CHECKING, cast
|
|
25
|
+
from unittest.mock import Mock
|
|
26
|
+
|
|
27
|
+
import dateutil
|
|
28
|
+
from cachetools.func import lru_cache
|
|
29
|
+
from fastapi.responses import ORJSONResponse, StreamingResponse
|
|
30
|
+
from pydantic import ValidationError as pydanticValidationError
|
|
31
|
+
from requests.models import Response as RequestsResponse
|
|
32
|
+
|
|
33
|
+
import eodag
|
|
34
|
+
from eodag import EOProduct
|
|
35
|
+
from eodag.api.product.metadata_mapping import (
|
|
36
|
+
NOT_AVAILABLE,
|
|
37
|
+
OFFLINE_STATUS,
|
|
38
|
+
ONLINE_STATUS,
|
|
39
|
+
OSEO_METADATA_MAPPING,
|
|
40
|
+
STAGING_STATUS,
|
|
41
|
+
)
|
|
42
|
+
from eodag.api.search_result import SearchResult
|
|
43
|
+
from eodag.config import load_stac_config
|
|
44
|
+
from eodag.plugins.crunch.filter_latest_intersect import FilterLatestIntersect
|
|
45
|
+
from eodag.plugins.crunch.filter_latest_tpl_name import FilterLatestByName
|
|
46
|
+
from eodag.plugins.crunch.filter_overlap import FilterOverlap
|
|
47
|
+
from eodag.rest.cache import cached
|
|
48
|
+
from eodag.rest.constants import (
|
|
49
|
+
CACHE_KEY_COLLECTION,
|
|
50
|
+
CACHE_KEY_COLLECTIONS,
|
|
51
|
+
CACHE_KEY_QUERYABLES,
|
|
52
|
+
)
|
|
53
|
+
from eodag.rest.stac import StacCatalog, StacCollection, StacCommon, StacItem
|
|
54
|
+
from eodag.rest.types.eodag_search import EODAGSearch
|
|
55
|
+
from eodag.rest.types.queryables import (
|
|
56
|
+
QueryablesGetParams,
|
|
57
|
+
StacQueryableProperty,
|
|
58
|
+
StacQueryables,
|
|
59
|
+
)
|
|
60
|
+
from eodag.rest.types.stac_search import SearchPostRequest
|
|
61
|
+
from eodag.rest.utils import (
|
|
62
|
+
Cruncher,
|
|
63
|
+
file_to_stream,
|
|
64
|
+
format_pydantic_error,
|
|
65
|
+
get_next_link,
|
|
66
|
+
)
|
|
67
|
+
from eodag.rest.utils.rfc3339 import rfc3339_str_to_datetime
|
|
68
|
+
from eodag.utils import (
|
|
69
|
+
_deprecated,
|
|
70
|
+
deepcopy,
|
|
71
|
+
dict_items_recursive_apply,
|
|
72
|
+
format_dict_items,
|
|
73
|
+
obj_md5sum,
|
|
74
|
+
urlencode,
|
|
75
|
+
)
|
|
76
|
+
from eodag.utils.exceptions import (
|
|
77
|
+
MisconfiguredError,
|
|
78
|
+
NotAvailableError,
|
|
79
|
+
ValidationError,
|
|
80
|
+
)
|
|
81
|
+
|
|
82
|
+
if TYPE_CHECKING:
|
|
83
|
+
from typing import Any, Callable, Dict, List, Optional, Tuple, Union
|
|
84
|
+
|
|
85
|
+
from fastapi import Request
|
|
86
|
+
from requests.auth import AuthBase
|
|
87
|
+
from starlette.responses import Response
|
|
88
|
+
|
|
89
|
+
|
|
90
|
+
eodag_api = eodag.EODataAccessGateway()
|
|
91
|
+
|
|
92
|
+
logger = logging.getLogger("eodag.rest.core")
|
|
93
|
+
|
|
94
|
+
stac_config = load_stac_config()
|
|
95
|
+
|
|
96
|
+
crunchers = {
|
|
97
|
+
"filterLatestIntersect": Cruncher(FilterLatestIntersect, []),
|
|
98
|
+
"filterLatestByName": Cruncher(FilterLatestByName, ["name_pattern"]),
|
|
99
|
+
"filterOverlap": Cruncher(FilterOverlap, ["minimum_overlap"]),
|
|
100
|
+
}
|
|
101
|
+
|
|
102
|
+
|
|
103
|
+
@_deprecated(reason="No more needed with STAC API + Swagger", version="2.6.1")
|
|
104
|
+
def get_home_page_content(base_url: str, ipp: Optional[int] = None) -> str:
|
|
105
|
+
"""Compute eodag service home page content
|
|
106
|
+
|
|
107
|
+
:param base_url: The service root URL
|
|
108
|
+
:type base_url: str
|
|
109
|
+
:param ipp: (optional) Items per page number
|
|
110
|
+
:type ipp: int
|
|
111
|
+
"""
|
|
112
|
+
base_url = base_url.rstrip("/") + "/"
|
|
113
|
+
content = f"""<h1>EODAG Server</h1><br />
|
|
114
|
+
<a href='{base_url}'>root</a><br />
|
|
115
|
+
<a href='{base_url}service-doc'>service-doc</a><br />
|
|
116
|
+
"""
|
|
117
|
+
return content
|
|
118
|
+
|
|
119
|
+
|
|
120
|
+
@_deprecated(
|
|
121
|
+
reason="Function internally used by get_home_page_content, also deprecated",
|
|
122
|
+
version="2.6.1",
|
|
123
|
+
)
|
|
124
|
+
def format_product_types(product_types: List[Dict[str, Any]]) -> str:
|
|
125
|
+
"""Format product_types
|
|
126
|
+
|
|
127
|
+
:param product_types: A list of EODAG product types as returned by the core api
|
|
128
|
+
:type product_types: list
|
|
129
|
+
"""
|
|
130
|
+
result: List[str] = []
|
|
131
|
+
for pt in product_types:
|
|
132
|
+
result.append(f'* *__{pt["ID"]}__*: {pt["abstract"]}')
|
|
133
|
+
return "\n".join(sorted(result))
|
|
134
|
+
|
|
135
|
+
|
|
136
|
+
def search_stac_items(
|
|
137
|
+
request: Request,
|
|
138
|
+
search_request: SearchPostRequest,
|
|
139
|
+
catalogs: Optional[List[str]] = None,
|
|
140
|
+
) -> Dict[str, Any]:
|
|
141
|
+
"""
|
|
142
|
+
Search and retrieve STAC items from the given catalogs.
|
|
143
|
+
|
|
144
|
+
This function takes a search request and optional catalogs list, performs a search using EODAG API, and returns a
|
|
145
|
+
dictionary of STAC items.
|
|
146
|
+
|
|
147
|
+
:param request: The incoming HTTP request with state information.
|
|
148
|
+
:type request: Request
|
|
149
|
+
:param search_request: The search criteria for STAC items.
|
|
150
|
+
:type search_request: SearchPostRequest
|
|
151
|
+
:param catalogs: (optional) A list of catalogs to search within. Defaults to None.
|
|
152
|
+
:type catalogs: Optional[List[str]]
|
|
153
|
+
:returns: A dictionary containing the STAC items and related metadata.
|
|
154
|
+
:rtype: Dict[str, Any]
|
|
155
|
+
|
|
156
|
+
The function handles the conversion of search criteria into STAC and EODAG compatible formats, validates the input
|
|
157
|
+
using pydantic, and constructs the appropriate URLs for querying the STAC API. It also manages pagination and the
|
|
158
|
+
construction of the 'next' link for the response.
|
|
159
|
+
|
|
160
|
+
If specific item IDs are provided, it retrieves the corresponding products. Otherwise, it performs a search based on
|
|
161
|
+
the provided criteria and time interval overlap checks.
|
|
162
|
+
|
|
163
|
+
The results are then formatted into STAC items and returned as part of the response dictionary, which includes the
|
|
164
|
+
items themselves, total count, and the next link if applicable.
|
|
165
|
+
"""
|
|
166
|
+
|
|
167
|
+
stac_args = search_request.model_dump(exclude_none=True)
|
|
168
|
+
if search_request.start_date:
|
|
169
|
+
stac_args["start_datetime"] = search_request.start_date
|
|
170
|
+
if search_request.end_date:
|
|
171
|
+
stac_args["end_datetime"] = search_request.end_date
|
|
172
|
+
if search_request.spatial_filter:
|
|
173
|
+
stac_args["geometry"] = search_request.spatial_filter
|
|
174
|
+
try:
|
|
175
|
+
eodag_args = EODAGSearch.model_validate(
|
|
176
|
+
stac_args, context={"isCatalog": bool(catalogs)}
|
|
177
|
+
)
|
|
178
|
+
except pydanticValidationError as e:
|
|
179
|
+
raise ValidationError(format_pydantic_error(e)) from e
|
|
180
|
+
|
|
181
|
+
catalog_url = re.sub("/items.*", "", request.state.url)
|
|
182
|
+
|
|
183
|
+
catalog = StacCatalog(
|
|
184
|
+
url=(
|
|
185
|
+
catalog_url
|
|
186
|
+
if catalogs
|
|
187
|
+
else catalog_url.replace(
|
|
188
|
+
"/search", f"/collections/{eodag_args.productType}"
|
|
189
|
+
)
|
|
190
|
+
),
|
|
191
|
+
stac_config=stac_config,
|
|
192
|
+
root=request.state.url_root,
|
|
193
|
+
provider=eodag_args.provider,
|
|
194
|
+
eodag_api=eodag_api,
|
|
195
|
+
catalogs=catalogs or [eodag_args.productType], # type: ignore
|
|
196
|
+
)
|
|
197
|
+
|
|
198
|
+
# get products by ids
|
|
199
|
+
if eodag_args.ids:
|
|
200
|
+
search_results = SearchResult([])
|
|
201
|
+
for item_id in eodag_args.ids:
|
|
202
|
+
sr = eodag_api.search(
|
|
203
|
+
id=item_id,
|
|
204
|
+
productType=catalogs[0] if catalogs else eodag_args.productType,
|
|
205
|
+
provider=eodag_args.provider,
|
|
206
|
+
)
|
|
207
|
+
search_results.extend(sr)
|
|
208
|
+
search_results.number_matched = len(search_results)
|
|
209
|
+
total = len(search_results)
|
|
210
|
+
|
|
211
|
+
elif time_interval_overlap(eodag_args, catalog):
|
|
212
|
+
criteria = {
|
|
213
|
+
**catalog.search_args,
|
|
214
|
+
**eodag_args.model_dump(exclude_none=True),
|
|
215
|
+
}
|
|
216
|
+
|
|
217
|
+
search_results = eodag_api.search(count=True, **criteria)
|
|
218
|
+
total = search_results.number_matched
|
|
219
|
+
if search_request.crunch:
|
|
220
|
+
search_results = crunch_products(
|
|
221
|
+
search_results, search_request.crunch, **criteria
|
|
222
|
+
)
|
|
223
|
+
else:
|
|
224
|
+
# return empty results
|
|
225
|
+
search_results = SearchResult([], 0)
|
|
226
|
+
total = 0
|
|
227
|
+
|
|
228
|
+
for record in search_results:
|
|
229
|
+
record.product_type = eodag_api.get_alias_from_product_type(record.product_type)
|
|
230
|
+
|
|
231
|
+
items = StacItem(
|
|
232
|
+
url=request.state.url,
|
|
233
|
+
stac_config=stac_config,
|
|
234
|
+
provider=eodag_args.provider,
|
|
235
|
+
eodag_api=eodag_api,
|
|
236
|
+
root=request.state.url_root,
|
|
237
|
+
).get_stac_items(
|
|
238
|
+
search_results=search_results,
|
|
239
|
+
total=total,
|
|
240
|
+
next_link=get_next_link(
|
|
241
|
+
request, search_request, total, eodag_args.items_per_page
|
|
242
|
+
),
|
|
243
|
+
catalog={
|
|
244
|
+
**catalog.data,
|
|
245
|
+
**{"url": catalog.url, "root": catalog.root},
|
|
246
|
+
},
|
|
247
|
+
)
|
|
248
|
+
return items
|
|
249
|
+
|
|
250
|
+
|
|
251
|
+
def download_stac_item(
|
|
252
|
+
request: Request,
|
|
253
|
+
catalogs: List[str],
|
|
254
|
+
item_id: str,
|
|
255
|
+
provider: Optional[str] = None,
|
|
256
|
+
asset: Optional[str] = None,
|
|
257
|
+
**kwargs: Any,
|
|
258
|
+
) -> Response:
|
|
259
|
+
"""Download item
|
|
260
|
+
|
|
261
|
+
:param catalogs: Catalogs list (only first is used as product_type)
|
|
262
|
+
:type catalogs: list
|
|
263
|
+
:param item_id: Product ID
|
|
264
|
+
:type item_id: str
|
|
265
|
+
:param provider: (optional) Chosen provider
|
|
266
|
+
:type provider: str
|
|
267
|
+
:param kwargs: additional download parameters
|
|
268
|
+
:type kwargs: Any
|
|
269
|
+
:returns: a stream of the downloaded data (zip file)
|
|
270
|
+
:rtype: Response
|
|
271
|
+
"""
|
|
272
|
+
product_type = catalogs[0]
|
|
273
|
+
|
|
274
|
+
search_results = eodag_api.search(
|
|
275
|
+
id=item_id, productType=product_type, provider=provider, **kwargs
|
|
276
|
+
)
|
|
277
|
+
if len(search_results) > 0:
|
|
278
|
+
product = cast(EOProduct, search_results[0])
|
|
279
|
+
|
|
280
|
+
else:
|
|
281
|
+
raise NotAvailableError(
|
|
282
|
+
f"Could not find {item_id} item in {product_type} collection"
|
|
283
|
+
+ (f" for provider {provider}" if provider else "")
|
|
284
|
+
)
|
|
285
|
+
auth = product.downloader_auth.authenticate() if product.downloader_auth else None
|
|
286
|
+
|
|
287
|
+
try:
|
|
288
|
+
if product.properties.get("orderLink"):
|
|
289
|
+
_order_and_update(product, auth, kwargs)
|
|
290
|
+
|
|
291
|
+
download_stream = product.downloader._stream_download_dict(
|
|
292
|
+
product,
|
|
293
|
+
auth=auth,
|
|
294
|
+
asset=asset,
|
|
295
|
+
wait=-1,
|
|
296
|
+
timeout=-1,
|
|
297
|
+
)
|
|
298
|
+
except NotImplementedError:
|
|
299
|
+
logger.warning(
|
|
300
|
+
"Download streaming not supported for %s: downloading locally then delete",
|
|
301
|
+
product.downloader,
|
|
302
|
+
)
|
|
303
|
+
download_stream = file_to_stream(
|
|
304
|
+
eodag_api.download(product, extract=False, asset=asset)
|
|
305
|
+
)
|
|
306
|
+
except NotAvailableError:
|
|
307
|
+
if product.properties.get("storageStatus") != ONLINE_STATUS:
|
|
308
|
+
kwargs["orderId"] = kwargs.get("orderId") or product.properties.get(
|
|
309
|
+
"orderId"
|
|
310
|
+
)
|
|
311
|
+
kwargs["provider"] = provider
|
|
312
|
+
qs = urlencode(kwargs, doseq=True)
|
|
313
|
+
download_link = f"{request.state.url}?{qs}"
|
|
314
|
+
return ORJSONResponse(
|
|
315
|
+
status_code=202,
|
|
316
|
+
headers={"Location": download_link},
|
|
317
|
+
content={
|
|
318
|
+
"description": "Product is not available yet, please try again using given updated location",
|
|
319
|
+
"status": product.properties.get("orderStatus"),
|
|
320
|
+
"location": download_link,
|
|
321
|
+
},
|
|
322
|
+
)
|
|
323
|
+
else:
|
|
324
|
+
raise
|
|
325
|
+
|
|
326
|
+
return StreamingResponse(
|
|
327
|
+
content=download_stream.content,
|
|
328
|
+
headers=download_stream.headers,
|
|
329
|
+
media_type=download_stream.media_type,
|
|
330
|
+
)
|
|
331
|
+
|
|
332
|
+
|
|
333
|
+
def _order_and_update(
|
|
334
|
+
product: EOProduct,
|
|
335
|
+
auth: Union[AuthBase, Dict[str, str], None],
|
|
336
|
+
query_args: Dict[str, Any],
|
|
337
|
+
) -> None:
|
|
338
|
+
"""Order product if needed and update given kwargs with order-status-dict"""
|
|
339
|
+
if product.properties.get("storageStatus") != ONLINE_STATUS and hasattr(
|
|
340
|
+
product.downloader, "order_response_process"
|
|
341
|
+
):
|
|
342
|
+
# update product (including orderStatusLink) if product was previously ordered
|
|
343
|
+
logger.debug("Use given download query arguments to parse order link")
|
|
344
|
+
response = Mock(spec=RequestsResponse)
|
|
345
|
+
response.status_code = 200
|
|
346
|
+
response.json.return_value = query_args
|
|
347
|
+
response.headers = {}
|
|
348
|
+
product.downloader.order_response_process(response, product)
|
|
349
|
+
|
|
350
|
+
if (
|
|
351
|
+
product.properties.get("storageStatus") != ONLINE_STATUS
|
|
352
|
+
and NOT_AVAILABLE in product.properties.get("orderStatusLink", "")
|
|
353
|
+
and hasattr(product.downloader, "orderDownload")
|
|
354
|
+
):
|
|
355
|
+
# first order
|
|
356
|
+
logger.debug("Order product")
|
|
357
|
+
order_status_dict = product.downloader.orderDownload(product=product, auth=auth)
|
|
358
|
+
query_args.update(order_status_dict or {})
|
|
359
|
+
|
|
360
|
+
if (
|
|
361
|
+
product.properties.get("storageStatus") == OFFLINE_STATUS
|
|
362
|
+
and product.properties.get("orderStatusLink")
|
|
363
|
+
and NOT_AVAILABLE not in product.properties.get("orderStatusLink", "")
|
|
364
|
+
):
|
|
365
|
+
product.properties["storageStatus"] = STAGING_STATUS
|
|
366
|
+
|
|
367
|
+
if product.properties.get("storageStatus") == STAGING_STATUS and hasattr(
|
|
368
|
+
product.downloader, "orderDownloadStatus"
|
|
369
|
+
):
|
|
370
|
+
# check order status if needed
|
|
371
|
+
logger.debug("Checking product order status")
|
|
372
|
+
product.downloader.orderDownloadStatus(product=product, auth=auth)
|
|
373
|
+
|
|
374
|
+
if product.properties.get("storageStatus") != ONLINE_STATUS:
|
|
375
|
+
raise NotAvailableError("Product is not available yet")
|
|
376
|
+
|
|
377
|
+
|
|
378
|
+
@lru_cache(maxsize=1)
|
|
379
|
+
def get_detailled_collections_list() -> List[Dict[str, Any]]:
|
|
380
|
+
"""Returns detailled collections / product_types list as a list of
|
|
381
|
+
config dicts
|
|
382
|
+
|
|
383
|
+
:returns: List of config dicts
|
|
384
|
+
:rtype: list
|
|
385
|
+
"""
|
|
386
|
+
return eodag_api.list_product_types(fetch_providers=False)
|
|
387
|
+
|
|
388
|
+
|
|
389
|
+
async def all_collections(
|
|
390
|
+
request: Request,
|
|
391
|
+
provider: Optional[str] = None,
|
|
392
|
+
q: Optional[str] = None,
|
|
393
|
+
platform: Optional[str] = None,
|
|
394
|
+
instrument: Optional[str] = None,
|
|
395
|
+
constellation: Optional[str] = None,
|
|
396
|
+
datetime: Optional[str] = None,
|
|
397
|
+
) -> Dict[str, Any]:
|
|
398
|
+
"""Build STAC collections
|
|
399
|
+
|
|
400
|
+
:param url: Requested URL
|
|
401
|
+
:type url: str
|
|
402
|
+
:param root: The API root
|
|
403
|
+
:type root: str
|
|
404
|
+
:param filters: Search collections filters
|
|
405
|
+
:type filters: CollectionsSearchRequest
|
|
406
|
+
:param provider: (optional) Chosen provider
|
|
407
|
+
:type provider: str
|
|
408
|
+
:returns: Collections dictionnary
|
|
409
|
+
:rtype: dict
|
|
410
|
+
"""
|
|
411
|
+
|
|
412
|
+
async def _fetch() -> Dict[str, Any]:
|
|
413
|
+
stac_collection = StacCollection(
|
|
414
|
+
url=request.state.url,
|
|
415
|
+
stac_config=stac_config,
|
|
416
|
+
provider=provider,
|
|
417
|
+
eodag_api=eodag_api,
|
|
418
|
+
root=request.state.url_root,
|
|
419
|
+
)
|
|
420
|
+
collections = deepcopy(stac_config["collections"])
|
|
421
|
+
collections["collections"] = stac_collection.get_collection_list(
|
|
422
|
+
q=q,
|
|
423
|
+
platform=platform,
|
|
424
|
+
instrument=instrument,
|
|
425
|
+
constellation=constellation,
|
|
426
|
+
datetime=datetime,
|
|
427
|
+
)
|
|
428
|
+
|
|
429
|
+
# # parse f-strings
|
|
430
|
+
format_args = deepcopy(stac_config)
|
|
431
|
+
format_args["collections"].update(
|
|
432
|
+
{"url": stac_collection.url, "root": stac_collection.root}
|
|
433
|
+
)
|
|
434
|
+
|
|
435
|
+
collections["links"] = [
|
|
436
|
+
format_dict_items(link, **format_args) for link in collections["links"]
|
|
437
|
+
]
|
|
438
|
+
|
|
439
|
+
collections = format_dict_items(collections, **format_args)
|
|
440
|
+
return collections
|
|
441
|
+
|
|
442
|
+
hashed_collections = hash(f"{provider}:{q}:{platform}:{instrument}:{constellation}")
|
|
443
|
+
cache_key = f"{CACHE_KEY_COLLECTIONS}:{hashed_collections}"
|
|
444
|
+
return await cached(_fetch, cache_key, request)
|
|
445
|
+
|
|
446
|
+
|
|
447
|
+
async def get_collection(
|
|
448
|
+
request: Request, collection_id: str, provider: Optional[str] = None
|
|
449
|
+
) -> Dict[str, Any]:
|
|
450
|
+
"""Build STAC collection by id
|
|
451
|
+
|
|
452
|
+
:param url: Requested URL
|
|
453
|
+
:type url: str
|
|
454
|
+
:param root: API root
|
|
455
|
+
:type root: str
|
|
456
|
+
:param collection_id: Product_type as ID of the collection
|
|
457
|
+
:type collection_id: str
|
|
458
|
+
:param provider: (optional) Chosen provider
|
|
459
|
+
:type provider: str
|
|
460
|
+
:returns: Collection dictionary
|
|
461
|
+
:rtype: dict
|
|
462
|
+
"""
|
|
463
|
+
|
|
464
|
+
async def _fetch() -> Dict[str, Any]:
|
|
465
|
+
stac_collection = StacCollection(
|
|
466
|
+
url=request.state.url,
|
|
467
|
+
stac_config=stac_config,
|
|
468
|
+
provider=provider,
|
|
469
|
+
eodag_api=eodag_api,
|
|
470
|
+
root=request.state.url_root,
|
|
471
|
+
)
|
|
472
|
+
collection_list = stac_collection.get_collection_list(collection=collection_id)
|
|
473
|
+
|
|
474
|
+
if not collection_list:
|
|
475
|
+
raise NotAvailableError(f"Collection {collection_id} does not exist.")
|
|
476
|
+
|
|
477
|
+
return collection_list[0]
|
|
478
|
+
|
|
479
|
+
cache_key = f"{CACHE_KEY_COLLECTION}:{provider}:{collection_id}"
|
|
480
|
+
return await cached(_fetch, cache_key, request)
|
|
481
|
+
|
|
482
|
+
|
|
483
|
+
async def get_stac_catalogs(
|
|
484
|
+
request: Request,
|
|
485
|
+
url: str,
|
|
486
|
+
catalogs: Optional[Tuple[str, ...]] = None,
|
|
487
|
+
provider: Optional[str] = None,
|
|
488
|
+
) -> Dict[str, Any]:
|
|
489
|
+
"""Build STAC catalog
|
|
490
|
+
|
|
491
|
+
:param url: Requested URL
|
|
492
|
+
:type url: str
|
|
493
|
+
:param root: (optional) API root
|
|
494
|
+
:type root: str
|
|
495
|
+
:param catalogs: (optional) Catalogs list
|
|
496
|
+
:type catalogs: list
|
|
497
|
+
:param provider: (optional) Chosen provider
|
|
498
|
+
:type provider: str
|
|
499
|
+
:returns: Catalog dictionary
|
|
500
|
+
:rtype: dict
|
|
501
|
+
"""
|
|
502
|
+
|
|
503
|
+
async def _fetch() -> Dict[str, Any]:
|
|
504
|
+
return StacCatalog(
|
|
505
|
+
url=url,
|
|
506
|
+
stac_config=stac_config,
|
|
507
|
+
root=request.state.url_root,
|
|
508
|
+
provider=provider,
|
|
509
|
+
eodag_api=eodag_api,
|
|
510
|
+
catalogs=list(catalogs) if catalogs else None,
|
|
511
|
+
).data
|
|
512
|
+
|
|
513
|
+
hashed_catalogs = hash(":".join(catalogs) if catalogs else None)
|
|
514
|
+
return await cached(
|
|
515
|
+
_fetch, f"{CACHE_KEY_COLLECTION}:{provider}:{hashed_catalogs}", request
|
|
516
|
+
)
|
|
517
|
+
|
|
518
|
+
|
|
519
|
+
def time_interval_overlap(eodag_args: EODAGSearch, catalog: StacCatalog) -> bool:
|
|
520
|
+
"""fix search date filter based on catalog date range"""
|
|
521
|
+
# check if time filtering appears both in search arguments and catalog
|
|
522
|
+
# (for catalogs built by date: i.e. `year/2020/month/05`)
|
|
523
|
+
if not set(["start", "end"]) <= set(eodag_args.model_dump().keys()) or not set(
|
|
524
|
+
["start", "end"]
|
|
525
|
+
) <= set(catalog.search_args.keys()):
|
|
526
|
+
return True
|
|
527
|
+
|
|
528
|
+
search_date_min = cast(
|
|
529
|
+
datetime.datetime,
|
|
530
|
+
(
|
|
531
|
+
dateutil.parser.parse(eodag_args.start) # type: ignore
|
|
532
|
+
if eodag_args.start
|
|
533
|
+
else datetime.datetime.min.replace(tzinfo=datetime.timezone.utc)
|
|
534
|
+
),
|
|
535
|
+
)
|
|
536
|
+
search_date_max = cast(
|
|
537
|
+
datetime.datetime,
|
|
538
|
+
(
|
|
539
|
+
dateutil.parser.parse(eodag_args.end) # type: ignore
|
|
540
|
+
if eodag_args.end
|
|
541
|
+
else datetime.datetime.now(tz=datetime.timezone.utc)
|
|
542
|
+
),
|
|
543
|
+
)
|
|
544
|
+
|
|
545
|
+
catalog_date_min = rfc3339_str_to_datetime(catalog.search_args["start"])
|
|
546
|
+
catalog_date_max = rfc3339_str_to_datetime(catalog.search_args["end"])
|
|
547
|
+
# check if date intervals overlap
|
|
548
|
+
if (search_date_min <= catalog_date_max) and (search_date_max >= catalog_date_min):
|
|
549
|
+
# use intersection
|
|
550
|
+
eodag_args.start = (
|
|
551
|
+
max(search_date_min, catalog_date_min).isoformat().replace("+00:00", "Z")
|
|
552
|
+
)
|
|
553
|
+
eodag_args.end = (
|
|
554
|
+
min(search_date_max, catalog_date_max).isoformat().replace("+00:00", "Z")
|
|
555
|
+
)
|
|
556
|
+
return True
|
|
557
|
+
|
|
558
|
+
logger.warning("Time intervals do not overlap")
|
|
559
|
+
return False
|
|
560
|
+
|
|
561
|
+
|
|
562
|
+
@lru_cache(maxsize=1)
|
|
563
|
+
def get_stac_conformance() -> Dict[str, str]:
|
|
564
|
+
"""Build STAC conformance
|
|
565
|
+
|
|
566
|
+
:returns: conformance dictionnary
|
|
567
|
+
:rtype: dict
|
|
568
|
+
"""
|
|
569
|
+
return stac_config["conformance"]
|
|
570
|
+
|
|
571
|
+
|
|
572
|
+
def get_stac_api_version() -> str:
|
|
573
|
+
"""Get STAC API version
|
|
574
|
+
|
|
575
|
+
:returns: STAC API version
|
|
576
|
+
:rtype: str
|
|
577
|
+
"""
|
|
578
|
+
return stac_config["stac_api_version"]
|
|
579
|
+
|
|
580
|
+
|
|
581
|
+
@lru_cache(maxsize=1)
|
|
582
|
+
def get_stac_extension_oseo(url: str) -> Dict[str, str]:
|
|
583
|
+
"""Build STAC OGC / OpenSearch Extension for EO
|
|
584
|
+
|
|
585
|
+
:param url: Requested URL
|
|
586
|
+
:type url: str
|
|
587
|
+
:returns: Catalog dictionnary
|
|
588
|
+
:rtype: dict
|
|
589
|
+
"""
|
|
590
|
+
|
|
591
|
+
apply_method: Callable[[str, str], str] = lambda _, x: str(x).replace(
|
|
592
|
+
"$.product.", "$."
|
|
593
|
+
)
|
|
594
|
+
item_mapping = dict_items_recursive_apply(stac_config["item"], apply_method)
|
|
595
|
+
|
|
596
|
+
# all properties as string type by default
|
|
597
|
+
oseo_properties = {
|
|
598
|
+
"oseo:{}".format(k): {
|
|
599
|
+
"type": "string",
|
|
600
|
+
"title": k[0].upper() + re.sub(r"([A-Z][a-z]+)", r" \1", k[1:]),
|
|
601
|
+
}
|
|
602
|
+
for k, v in OSEO_METADATA_MAPPING.items()
|
|
603
|
+
if v not in str(item_mapping)
|
|
604
|
+
}
|
|
605
|
+
|
|
606
|
+
return StacCommon.get_stac_extension(
|
|
607
|
+
url=url, stac_config=stac_config, extension="oseo", properties=oseo_properties
|
|
608
|
+
)
|
|
609
|
+
|
|
610
|
+
|
|
611
|
+
async def get_queryables(
|
|
612
|
+
request: Request,
|
|
613
|
+
params: QueryablesGetParams,
|
|
614
|
+
provider: Optional[str] = None,
|
|
615
|
+
) -> Dict[str, Any]:
|
|
616
|
+
"""Fetch the queryable properties for a collection.
|
|
617
|
+
|
|
618
|
+
:param collection_id: The ID of the collection.
|
|
619
|
+
:type collection_id: str
|
|
620
|
+
:returns: A set containing the STAC standardized queryable properties for a collection.
|
|
621
|
+
:rtype Dict[str, StacQueryableProperty]: set
|
|
622
|
+
"""
|
|
623
|
+
|
|
624
|
+
async def _fetch() -> Dict[str, Any]:
|
|
625
|
+
python_queryables = eodag_api.list_queryables(
|
|
626
|
+
provider=provider, **params.model_dump(exclude_none=True, by_alias=True)
|
|
627
|
+
)
|
|
628
|
+
python_queryables.pop("start")
|
|
629
|
+
python_queryables.pop("end")
|
|
630
|
+
|
|
631
|
+
# productType and id are already default in stac collection and id
|
|
632
|
+
python_queryables.pop("productType", None)
|
|
633
|
+
python_queryables.pop("id", None)
|
|
634
|
+
|
|
635
|
+
stac_queryables: Dict[str, StacQueryableProperty] = deepcopy(
|
|
636
|
+
StacQueryables.default_properties
|
|
637
|
+
)
|
|
638
|
+
for param, queryable in python_queryables.items():
|
|
639
|
+
stac_param = EODAGSearch.to_stac(param)
|
|
640
|
+
# only keep "datetime" queryable for dates
|
|
641
|
+
if stac_param in stac_queryables or stac_param in (
|
|
642
|
+
"start_datetime",
|
|
643
|
+
"end_datetime",
|
|
644
|
+
):
|
|
645
|
+
continue
|
|
646
|
+
|
|
647
|
+
stac_queryables[
|
|
648
|
+
stac_param
|
|
649
|
+
] = StacQueryableProperty.from_python_field_definition(
|
|
650
|
+
stac_param, queryable
|
|
651
|
+
)
|
|
652
|
+
|
|
653
|
+
if params.collection:
|
|
654
|
+
stac_queryables.pop("collection")
|
|
655
|
+
|
|
656
|
+
return StacQueryables(
|
|
657
|
+
q_id=request.state.url,
|
|
658
|
+
additional_properties=bool(not params.collection),
|
|
659
|
+
properties=stac_queryables,
|
|
660
|
+
).model_dump(mode="json", by_alias=True)
|
|
661
|
+
|
|
662
|
+
hashed_queryables = hash(params.model_dump_json())
|
|
663
|
+
return await cached(
|
|
664
|
+
_fetch, f"{CACHE_KEY_QUERYABLES}:{provider}:{hashed_queryables}", request
|
|
665
|
+
)
|
|
666
|
+
|
|
667
|
+
|
|
668
|
+
@_deprecated(
|
|
669
|
+
reason="Used to format output from deprecated function get_home_page_content",
|
|
670
|
+
version="2.6.1",
|
|
671
|
+
)
|
|
672
|
+
def get_templates_path() -> str:
|
|
673
|
+
"""Returns Jinja templates path"""
|
|
674
|
+
return os.path.join(os.path.dirname(__file__), "templates")
|
|
675
|
+
|
|
676
|
+
|
|
677
|
+
def crunch_products(
|
|
678
|
+
products: SearchResult, cruncher_name: str, **kwargs: Any
|
|
679
|
+
) -> SearchResult:
|
|
680
|
+
"""Apply an eodag cruncher to filter products"""
|
|
681
|
+
cruncher = crunchers.get(cruncher_name)
|
|
682
|
+
if not cruncher:
|
|
683
|
+
raise ValidationError(
|
|
684
|
+
f'Unknown crunch name. Use one of: {", ".join(crunchers.keys())}'
|
|
685
|
+
)
|
|
686
|
+
|
|
687
|
+
cruncher_config: Dict[str, Any] = {}
|
|
688
|
+
for config_param in cruncher.config_params:
|
|
689
|
+
config_param_value = kwargs.get(config_param)
|
|
690
|
+
if not config_param_value:
|
|
691
|
+
raise ValidationError(
|
|
692
|
+
(
|
|
693
|
+
f"cruncher {cruncher} require additional parameters:"
|
|
694
|
+
f' {", ".join(cruncher.config_params)}'
|
|
695
|
+
)
|
|
696
|
+
)
|
|
697
|
+
cruncher_config[config_param] = config_param_value
|
|
698
|
+
|
|
699
|
+
try:
|
|
700
|
+
products = products.crunch(cruncher.clazz(cruncher_config), **kwargs)
|
|
701
|
+
except MisconfiguredError as e:
|
|
702
|
+
raise ValidationError(str(e)) from e
|
|
703
|
+
|
|
704
|
+
return products
|
|
705
|
+
|
|
706
|
+
|
|
707
|
+
def eodag_api_init() -> None:
|
|
708
|
+
"""Init EODataAccessGateway server instance, pre-running all time consuming tasks"""
|
|
709
|
+
eodag_api.fetch_product_types_list()
|
|
710
|
+
StacCollection.fetch_external_stac_collections(eodag_api)
|
|
711
|
+
|
|
712
|
+
# update eodag product_types config form external stac collections
|
|
713
|
+
for p, p_f in eodag_api.product_types_config.source.items():
|
|
714
|
+
for key in (p, p_f.get("alias")):
|
|
715
|
+
if key is None:
|
|
716
|
+
continue
|
|
717
|
+
ext_col = StacCollection.ext_stac_collections.get(key)
|
|
718
|
+
if not ext_col:
|
|
719
|
+
continue
|
|
720
|
+
platform: Union[str, List[str]] = ext_col.get("summaries", {}).get(
|
|
721
|
+
"platform"
|
|
722
|
+
)
|
|
723
|
+
constellation: Union[str, List[str]] = ext_col.get("summaries", {}).get(
|
|
724
|
+
"constellation"
|
|
725
|
+
)
|
|
726
|
+
# Check if platform or constellation are lists and join them into a string if they are
|
|
727
|
+
if isinstance(platform, list):
|
|
728
|
+
platform = ",".join(platform)
|
|
729
|
+
if isinstance(constellation, list):
|
|
730
|
+
constellation = ",".join(constellation)
|
|
731
|
+
|
|
732
|
+
update_fields = {
|
|
733
|
+
"title": ext_col.get("title"),
|
|
734
|
+
"abstract": ext_col["description"],
|
|
735
|
+
"keywords": ext_col.get("keywords"),
|
|
736
|
+
"instrument": ",".join(
|
|
737
|
+
ext_col.get("summaries", {}).get("instruments", [])
|
|
738
|
+
),
|
|
739
|
+
"platform": constellation,
|
|
740
|
+
"platformSerialIdentifier": platform,
|
|
741
|
+
"processingLevel": ext_col.get("summaries", {}).get("processing:level"),
|
|
742
|
+
"license": ext_col["license"],
|
|
743
|
+
"missionStartDate": ext_col["extent"]["temporal"]["interval"][0][0],
|
|
744
|
+
"missionEndDate": ext_col["extent"]["temporal"]["interval"][-1][1],
|
|
745
|
+
}
|
|
746
|
+
clean = {k: v for k, v in update_fields.items() if v}
|
|
747
|
+
p_f.update(clean)
|
|
748
|
+
|
|
749
|
+
eodag_api.product_types_config_md5 = obj_md5sum(
|
|
750
|
+
eodag_api.product_types_config.source
|
|
751
|
+
)
|
|
752
|
+
|
|
753
|
+
eodag_api.build_index()
|
|
754
|
+
|
|
755
|
+
# pre-build search plugins
|
|
756
|
+
for provider in eodag_api.available_providers():
|
|
757
|
+
next(eodag_api._plugins_manager.get_search_plugins(provider=provider))
|