eodag 2.12.1__py3-none-any.whl → 3.0.0b1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- eodag/api/core.py +434 -319
- eodag/api/product/__init__.py +5 -1
- eodag/api/product/_assets.py +7 -2
- eodag/api/product/_product.py +46 -68
- eodag/api/product/metadata_mapping.py +181 -66
- eodag/api/search_result.py +21 -1
- eodag/cli.py +20 -6
- eodag/config.py +95 -6
- eodag/plugins/apis/base.py +8 -165
- eodag/plugins/apis/ecmwf.py +36 -24
- eodag/plugins/apis/usgs.py +40 -24
- eodag/plugins/authentication/aws_auth.py +2 -2
- eodag/plugins/authentication/header.py +31 -6
- eodag/plugins/authentication/keycloak.py +13 -84
- eodag/plugins/authentication/oauth.py +3 -3
- eodag/plugins/authentication/openid_connect.py +256 -46
- eodag/plugins/authentication/qsauth.py +3 -0
- eodag/plugins/authentication/sas_auth.py +8 -1
- eodag/plugins/authentication/token.py +92 -46
- eodag/plugins/authentication/token_exchange.py +120 -0
- eodag/plugins/download/aws.py +86 -91
- eodag/plugins/download/base.py +72 -40
- eodag/plugins/download/http.py +607 -264
- eodag/plugins/download/s3rest.py +28 -15
- eodag/plugins/manager.py +73 -57
- eodag/plugins/search/__init__.py +36 -0
- eodag/plugins/search/base.py +225 -18
- eodag/plugins/search/build_search_result.py +389 -32
- eodag/plugins/search/cop_marine.py +378 -0
- eodag/plugins/search/creodias_s3.py +15 -14
- eodag/plugins/search/csw.py +5 -7
- eodag/plugins/search/data_request_search.py +44 -20
- eodag/plugins/search/qssearch.py +508 -203
- eodag/plugins/search/static_stac_search.py +99 -36
- eodag/resources/constraints/climate-dt.json +13 -0
- eodag/resources/constraints/extremes-dt.json +8 -0
- eodag/resources/ext_product_types.json +1 -1
- eodag/resources/product_types.yml +1897 -34
- eodag/resources/providers.yml +3539 -3277
- eodag/resources/stac.yml +48 -54
- eodag/resources/stac_api.yml +71 -25
- eodag/resources/stac_provider.yml +5 -0
- eodag/resources/user_conf_template.yml +51 -3
- eodag/rest/__init__.py +6 -0
- eodag/rest/cache.py +70 -0
- eodag/rest/config.py +68 -0
- eodag/rest/constants.py +27 -0
- eodag/rest/core.py +757 -0
- eodag/rest/server.py +397 -258
- eodag/rest/stac.py +438 -307
- eodag/rest/types/collections_search.py +44 -0
- eodag/rest/types/eodag_search.py +232 -43
- eodag/rest/types/{stac_queryables.py → queryables.py} +81 -43
- eodag/rest/types/stac_search.py +277 -0
- eodag/rest/utils/__init__.py +216 -0
- eodag/rest/utils/cql_evaluate.py +119 -0
- eodag/rest/utils/rfc3339.py +65 -0
- eodag/types/__init__.py +99 -9
- eodag/types/bbox.py +15 -14
- eodag/types/download_args.py +31 -0
- eodag/types/search_args.py +58 -7
- eodag/types/whoosh.py +81 -0
- eodag/utils/__init__.py +72 -9
- eodag/utils/constraints.py +37 -37
- eodag/utils/exceptions.py +23 -17
- eodag/utils/requests.py +138 -0
- eodag/utils/rest.py +104 -0
- eodag/utils/stac_reader.py +100 -16
- {eodag-2.12.1.dist-info → eodag-3.0.0b1.dist-info}/METADATA +64 -44
- eodag-3.0.0b1.dist-info/RECORD +109 -0
- {eodag-2.12.1.dist-info → eodag-3.0.0b1.dist-info}/WHEEL +1 -1
- {eodag-2.12.1.dist-info → eodag-3.0.0b1.dist-info}/entry_points.txt +6 -5
- eodag/plugins/apis/cds.py +0 -540
- eodag/rest/utils.py +0 -1133
- eodag-2.12.1.dist-info/RECORD +0 -94
- {eodag-2.12.1.dist-info → eodag-3.0.0b1.dist-info}/LICENSE +0 -0
- {eodag-2.12.1.dist-info → eodag-3.0.0b1.dist-info}/top_level.txt +0 -0
eodag/rest/utils.py
DELETED
|
@@ -1,1133 +0,0 @@
|
|
|
1
|
-
# -*- coding: utf-8 -*-
|
|
2
|
-
# Copyright 2018, CS Systemes d'Information, https://www.csgroup.eu/
|
|
3
|
-
#
|
|
4
|
-
# This file is part of EODAG project
|
|
5
|
-
# https://www.github.com/CS-SI/EODAG
|
|
6
|
-
#
|
|
7
|
-
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
8
|
-
# you may not use this file except in compliance with the License.
|
|
9
|
-
# You may obtain a copy of the License at
|
|
10
|
-
#
|
|
11
|
-
# http://www.apache.org/licenses/LICENSE-2.0
|
|
12
|
-
#
|
|
13
|
-
# Unless required by applicable law or agreed to in writing, software
|
|
14
|
-
# distributed under the License is distributed on an "AS IS" BASIS,
|
|
15
|
-
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
16
|
-
# See the License for the specific language governing permissions and
|
|
17
|
-
# limitations under the License.
|
|
18
|
-
from __future__ import annotations
|
|
19
|
-
|
|
20
|
-
import ast
|
|
21
|
-
import datetime
|
|
22
|
-
import glob
|
|
23
|
-
import json
|
|
24
|
-
import logging
|
|
25
|
-
import os
|
|
26
|
-
import re
|
|
27
|
-
from shutil import make_archive, rmtree
|
|
28
|
-
from typing import (
|
|
29
|
-
TYPE_CHECKING,
|
|
30
|
-
Any,
|
|
31
|
-
Callable,
|
|
32
|
-
Dict,
|
|
33
|
-
Iterator,
|
|
34
|
-
List,
|
|
35
|
-
NamedTuple,
|
|
36
|
-
Optional,
|
|
37
|
-
Tuple,
|
|
38
|
-
Union,
|
|
39
|
-
)
|
|
40
|
-
from urllib.parse import urlencode
|
|
41
|
-
|
|
42
|
-
import dateutil.parser
|
|
43
|
-
from dateutil import tz
|
|
44
|
-
from fastapi.responses import StreamingResponse
|
|
45
|
-
from shapely.geometry import Polygon, shape
|
|
46
|
-
|
|
47
|
-
import eodag
|
|
48
|
-
from eodag import EOProduct
|
|
49
|
-
from eodag.api.product.metadata_mapping import OSEO_METADATA_MAPPING
|
|
50
|
-
from eodag.api.search_result import SearchResult
|
|
51
|
-
from eodag.config import load_stac_config, load_stac_provider_config
|
|
52
|
-
from eodag.plugins.crunch.filter_latest_intersect import FilterLatestIntersect
|
|
53
|
-
from eodag.plugins.crunch.filter_latest_tpl_name import FilterLatestByName
|
|
54
|
-
from eodag.plugins.crunch.filter_overlap import FilterOverlap
|
|
55
|
-
from eodag.rest.stac import StacCatalog, StacCollection, StacCommon, StacItem
|
|
56
|
-
from eodag.rest.types.eodag_search import EODAGSearch
|
|
57
|
-
from eodag.rest.types.stac_queryables import StacQueryableProperty
|
|
58
|
-
from eodag.utils import (
|
|
59
|
-
DEFAULT_ITEMS_PER_PAGE,
|
|
60
|
-
DEFAULT_PAGE,
|
|
61
|
-
GENERIC_PRODUCT_TYPE,
|
|
62
|
-
_deprecated,
|
|
63
|
-
dict_items_recursive_apply,
|
|
64
|
-
string_to_jsonpath,
|
|
65
|
-
)
|
|
66
|
-
from eodag.utils.exceptions import (
|
|
67
|
-
MisconfiguredError,
|
|
68
|
-
NoMatchingProductType,
|
|
69
|
-
NotAvailableError,
|
|
70
|
-
RequestError,
|
|
71
|
-
UnsupportedProductType,
|
|
72
|
-
ValidationError,
|
|
73
|
-
)
|
|
74
|
-
|
|
75
|
-
if TYPE_CHECKING:
|
|
76
|
-
from io import BufferedReader
|
|
77
|
-
|
|
78
|
-
from shapely.geometry.base import BaseGeometry
|
|
79
|
-
|
|
80
|
-
|
|
81
|
-
logger = logging.getLogger("eodag.rest.utils")
|
|
82
|
-
|
|
83
|
-
eodag_api = eodag.EODataAccessGateway()
|
|
84
|
-
|
|
85
|
-
|
|
86
|
-
class Cruncher(NamedTuple):
|
|
87
|
-
"""Type hinted Cruncher namedTuple"""
|
|
88
|
-
|
|
89
|
-
clazz: Callable[..., Any]
|
|
90
|
-
config_params: List[str]
|
|
91
|
-
|
|
92
|
-
|
|
93
|
-
crunchers = {
|
|
94
|
-
"latestIntersect": Cruncher(FilterLatestIntersect, []),
|
|
95
|
-
"latestByName": Cruncher(FilterLatestByName, ["name_pattern"]),
|
|
96
|
-
"overlap": Cruncher(FilterOverlap, ["minimum_overlap"]),
|
|
97
|
-
}
|
|
98
|
-
stac_config = load_stac_config()
|
|
99
|
-
stac_provider_config = load_stac_provider_config()
|
|
100
|
-
|
|
101
|
-
STAC_QUERY_PATTERN = "query.*.*"
|
|
102
|
-
|
|
103
|
-
|
|
104
|
-
@_deprecated(
|
|
105
|
-
reason="Function internally used by get_home_page_content, also deprecated",
|
|
106
|
-
version="2.6.1",
|
|
107
|
-
)
|
|
108
|
-
def format_product_types(product_types: List[Dict[str, Any]]) -> str:
|
|
109
|
-
"""Format product_types
|
|
110
|
-
|
|
111
|
-
:param product_types: A list of EODAG product types as returned by the core api
|
|
112
|
-
:type product_types: list
|
|
113
|
-
"""
|
|
114
|
-
result: List[str] = []
|
|
115
|
-
for pt in product_types:
|
|
116
|
-
result.append("* *__{ID}__*: {abstract}".format(**pt))
|
|
117
|
-
return "\n".join(sorted(result))
|
|
118
|
-
|
|
119
|
-
|
|
120
|
-
def get_detailled_collections_list(
|
|
121
|
-
provider: Optional[str] = None, fetch_providers: bool = True
|
|
122
|
-
) -> List[Dict[str, Any]]:
|
|
123
|
-
"""Returns detailled collections / product_types list for a given provider as a list of config dicts
|
|
124
|
-
|
|
125
|
-
:param provider: (optional) Chosen provider
|
|
126
|
-
:type provider: str
|
|
127
|
-
:param fetch_providers: (optional) Whether to fetch providers for new product
|
|
128
|
-
types or not
|
|
129
|
-
:type fetch_providers: bool
|
|
130
|
-
:returns: List of config dicts
|
|
131
|
-
:rtype: list
|
|
132
|
-
"""
|
|
133
|
-
return eodag_api.list_product_types(
|
|
134
|
-
provider=provider, fetch_providers=fetch_providers
|
|
135
|
-
)
|
|
136
|
-
|
|
137
|
-
|
|
138
|
-
@_deprecated(reason="No more needed with STAC API + Swagger", version="2.6.1")
|
|
139
|
-
def get_home_page_content(base_url: str, ipp: Optional[int] = None) -> str:
|
|
140
|
-
"""Compute eodag service home page content
|
|
141
|
-
|
|
142
|
-
:param base_url: The service root URL
|
|
143
|
-
:type base_url: str
|
|
144
|
-
:param ipp: (optional) Items per page number
|
|
145
|
-
:type ipp: int
|
|
146
|
-
"""
|
|
147
|
-
base_url = base_url.rstrip("/") + "/"
|
|
148
|
-
content = f"""<h1>EODAG Server</h1><br />
|
|
149
|
-
<a href='{base_url}'>root</a><br />
|
|
150
|
-
<a href='{base_url}service-doc'>service-doc</a><br />
|
|
151
|
-
"""
|
|
152
|
-
return content
|
|
153
|
-
|
|
154
|
-
|
|
155
|
-
@_deprecated(
|
|
156
|
-
reason="Used to format output from deprecated function get_home_page_content",
|
|
157
|
-
version="2.6.1",
|
|
158
|
-
)
|
|
159
|
-
def get_templates_path() -> str:
|
|
160
|
-
"""Returns Jinja templates path"""
|
|
161
|
-
return os.path.join(os.path.dirname(__file__), "templates")
|
|
162
|
-
|
|
163
|
-
|
|
164
|
-
def get_product_types(
|
|
165
|
-
provider: Optional[str] = None, filters: Optional[Dict[str, Any]] = None
|
|
166
|
-
) -> List[Dict[str, Any]]:
|
|
167
|
-
"""Returns a list of supported product types
|
|
168
|
-
|
|
169
|
-
:param provider: (optional) Provider name
|
|
170
|
-
:type provider: str
|
|
171
|
-
:param filters: (optional) Additional filters for product types search
|
|
172
|
-
:type filters: dict
|
|
173
|
-
:returns: A list of corresponding product types
|
|
174
|
-
:rtype: list
|
|
175
|
-
"""
|
|
176
|
-
if filters is None:
|
|
177
|
-
filters = {}
|
|
178
|
-
try:
|
|
179
|
-
guessed_product_types = eodag_api.guess_product_type(
|
|
180
|
-
instrument=filters.get("instrument"),
|
|
181
|
-
platform=filters.get("platform"),
|
|
182
|
-
platformSerialIdentifier=filters.get("platformSerialIdentifier"),
|
|
183
|
-
sensorType=filters.get("sensorType"),
|
|
184
|
-
processingLevel=filters.get("processingLevel"),
|
|
185
|
-
)
|
|
186
|
-
except NoMatchingProductType:
|
|
187
|
-
guessed_product_types = []
|
|
188
|
-
if guessed_product_types:
|
|
189
|
-
product_types = [
|
|
190
|
-
pt
|
|
191
|
-
for pt in eodag_api.list_product_types(provider=provider)
|
|
192
|
-
if pt["ID"] in guessed_product_types
|
|
193
|
-
]
|
|
194
|
-
else:
|
|
195
|
-
product_types = eodag_api.list_product_types(provider=provider)
|
|
196
|
-
return product_types
|
|
197
|
-
|
|
198
|
-
|
|
199
|
-
def search_bbox(request_bbox: str) -> Optional[Dict[str, float]]:
|
|
200
|
-
"""Transform request bounding box as a bbox suitable for eodag search"""
|
|
201
|
-
|
|
202
|
-
eodag_bbox = None
|
|
203
|
-
search_bbox_keys = ["lonmin", "latmin", "lonmax", "latmax"]
|
|
204
|
-
|
|
205
|
-
if not request_bbox:
|
|
206
|
-
return None
|
|
207
|
-
|
|
208
|
-
try:
|
|
209
|
-
request_bbox_list = [float(coord) for coord in request_bbox.split(",")]
|
|
210
|
-
except ValueError as e:
|
|
211
|
-
raise ValidationError("invalid box coordinate type: %s" % e)
|
|
212
|
-
|
|
213
|
-
eodag_bbox = dict(zip(search_bbox_keys, request_bbox_list))
|
|
214
|
-
if len(eodag_bbox) != 4:
|
|
215
|
-
raise ValidationError("input box is invalid: %s" % request_bbox)
|
|
216
|
-
|
|
217
|
-
return eodag_bbox
|
|
218
|
-
|
|
219
|
-
|
|
220
|
-
def get_date(date: Optional[str]) -> Optional[str]:
|
|
221
|
-
"""Check if the input date can be parsed as a date"""
|
|
222
|
-
|
|
223
|
-
if not date:
|
|
224
|
-
return None
|
|
225
|
-
try:
|
|
226
|
-
return (
|
|
227
|
-
dateutil.parser.parse(date)
|
|
228
|
-
.replace(tzinfo=tz.UTC)
|
|
229
|
-
.isoformat()
|
|
230
|
-
.replace("+00:00", "")
|
|
231
|
-
)
|
|
232
|
-
except ValueError as e:
|
|
233
|
-
exc = ValidationError("invalid input date: %s" % e)
|
|
234
|
-
raise exc
|
|
235
|
-
|
|
236
|
-
|
|
237
|
-
def get_int(input: Optional[Any]) -> Optional[int]:
|
|
238
|
-
"""Check if the input can be parsed as an integer"""
|
|
239
|
-
|
|
240
|
-
if input is None:
|
|
241
|
-
return None
|
|
242
|
-
|
|
243
|
-
try:
|
|
244
|
-
val = int(input)
|
|
245
|
-
except ValueError as e:
|
|
246
|
-
raise ValidationError("invalid input integer value: %s" % e)
|
|
247
|
-
return val
|
|
248
|
-
|
|
249
|
-
|
|
250
|
-
def filter_products(
|
|
251
|
-
products: SearchResult, arguments: Dict[str, Any], **kwargs: Any
|
|
252
|
-
) -> SearchResult:
|
|
253
|
-
"""Apply an eodag cruncher to filter products"""
|
|
254
|
-
filter_name = arguments.get("filter")
|
|
255
|
-
if filter_name:
|
|
256
|
-
cruncher = crunchers.get(filter_name)
|
|
257
|
-
if not cruncher:
|
|
258
|
-
raise ValidationError("unknown filter name")
|
|
259
|
-
|
|
260
|
-
cruncher_config: Dict[str, Any] = dict()
|
|
261
|
-
for config_param in cruncher.config_params:
|
|
262
|
-
config_param_value = arguments.get(config_param)
|
|
263
|
-
if not config_param_value:
|
|
264
|
-
raise ValidationError(
|
|
265
|
-
"filter additional parameters required: %s"
|
|
266
|
-
% ", ".join(cruncher.config_params)
|
|
267
|
-
)
|
|
268
|
-
cruncher_config[config_param] = config_param_value
|
|
269
|
-
|
|
270
|
-
try:
|
|
271
|
-
products = products.crunch(cruncher.clazz(cruncher_config), **kwargs)
|
|
272
|
-
except MisconfiguredError as e:
|
|
273
|
-
raise ValidationError(str(e))
|
|
274
|
-
|
|
275
|
-
return products
|
|
276
|
-
|
|
277
|
-
|
|
278
|
-
def get_pagination_info(
|
|
279
|
-
arguments: Dict[str, Any]
|
|
280
|
-
) -> Tuple[Optional[int], Optional[int]]:
|
|
281
|
-
"""Get pagination arguments"""
|
|
282
|
-
page = get_int(arguments.pop("page", DEFAULT_PAGE))
|
|
283
|
-
# items_per_page can be specified using limit or itemsPerPage
|
|
284
|
-
items_per_page = get_int(arguments.pop("limit", DEFAULT_ITEMS_PER_PAGE))
|
|
285
|
-
items_per_page = get_int(arguments.pop("itemsPerPage", items_per_page))
|
|
286
|
-
|
|
287
|
-
if page is not None and page < 0:
|
|
288
|
-
raise ValidationError("invalid page number. Must be positive integer")
|
|
289
|
-
if items_per_page is not None and items_per_page < 0:
|
|
290
|
-
raise ValidationError(
|
|
291
|
-
"invalid number of items per page. Must be positive integer"
|
|
292
|
-
)
|
|
293
|
-
return page, items_per_page
|
|
294
|
-
|
|
295
|
-
|
|
296
|
-
def get_geometry(arguments: Dict[str, Any]) -> Optional[BaseGeometry]:
|
|
297
|
-
"""Get geometry from arguments"""
|
|
298
|
-
if arguments.get("intersects") and arguments.get("bbox"):
|
|
299
|
-
raise ValidationError("Only one of bbox and intersects can be used at a time.")
|
|
300
|
-
|
|
301
|
-
if arguments.get("bbox"):
|
|
302
|
-
request_bbox = arguments.pop("bbox")
|
|
303
|
-
if isinstance(request_bbox, str):
|
|
304
|
-
request_bbox = request_bbox.split(",")
|
|
305
|
-
elif not isinstance(request_bbox, list):
|
|
306
|
-
raise ValidationError("bbox argument type should be Array")
|
|
307
|
-
|
|
308
|
-
try:
|
|
309
|
-
request_bbox = [float(coord) for coord in request_bbox]
|
|
310
|
-
except ValueError as e:
|
|
311
|
-
raise ValidationError(f"invalid bbox coordinate type: {e}")
|
|
312
|
-
|
|
313
|
-
if len(request_bbox) == 4:
|
|
314
|
-
min_x, min_y, max_x, max_y = request_bbox
|
|
315
|
-
elif len(request_bbox) == 6:
|
|
316
|
-
min_x, min_y, _, max_x, max_y, _ = request_bbox
|
|
317
|
-
else:
|
|
318
|
-
raise ValidationError(
|
|
319
|
-
f"invalid bbox length ({len(request_bbox)}) for bbox {request_bbox}"
|
|
320
|
-
)
|
|
321
|
-
|
|
322
|
-
geom = Polygon([(min_x, min_y), (min_x, max_y), (max_x, max_y), (max_x, min_y)])
|
|
323
|
-
|
|
324
|
-
elif arguments.get("intersects"):
|
|
325
|
-
intersects_value = arguments.pop("intersects")
|
|
326
|
-
if isinstance(intersects_value, str):
|
|
327
|
-
try:
|
|
328
|
-
intersects_dict = json.loads(intersects_value)
|
|
329
|
-
except json.JSONDecodeError:
|
|
330
|
-
raise ValidationError(
|
|
331
|
-
"The 'intersects' parameter is not a valid JSON string."
|
|
332
|
-
)
|
|
333
|
-
else:
|
|
334
|
-
intersects_dict = intersects_value
|
|
335
|
-
|
|
336
|
-
try:
|
|
337
|
-
geom = shape(intersects_dict)
|
|
338
|
-
except Exception as e:
|
|
339
|
-
raise ValidationError(
|
|
340
|
-
f"The 'intersects' parameter does not represent a valid geometry: {str(e)}"
|
|
341
|
-
)
|
|
342
|
-
|
|
343
|
-
else:
|
|
344
|
-
geom = None
|
|
345
|
-
|
|
346
|
-
return geom
|
|
347
|
-
|
|
348
|
-
|
|
349
|
-
def get_datetime(arguments: Dict[str, Any]) -> Tuple[Optional[str], Optional[str]]:
|
|
350
|
-
"""Get the datetime criterias from the search arguments
|
|
351
|
-
|
|
352
|
-
:param arguments: Request args
|
|
353
|
-
:type arguments: dict
|
|
354
|
-
:returns: Start date and end date from datetime string.
|
|
355
|
-
:rtype: Tuple[Optional[str], Optional[str]]
|
|
356
|
-
"""
|
|
357
|
-
datetime_str = arguments.pop("datetime", None)
|
|
358
|
-
|
|
359
|
-
if datetime_str:
|
|
360
|
-
datetime_split = datetime_str.split("/")
|
|
361
|
-
if len(datetime_split) > 1:
|
|
362
|
-
dtstart = datetime_split[0] if datetime_split[0] != ".." else None
|
|
363
|
-
dtend = datetime_split[1] if datetime_split[1] != ".." else None
|
|
364
|
-
elif len(datetime_split) == 1:
|
|
365
|
-
# same time for start & end if only one is given
|
|
366
|
-
dtstart, dtend = datetime_split[0:1] * 2
|
|
367
|
-
else:
|
|
368
|
-
return None, None
|
|
369
|
-
|
|
370
|
-
return get_date(dtstart), get_date(dtend)
|
|
371
|
-
|
|
372
|
-
else:
|
|
373
|
-
# return already set (dtstart, dtend) or None
|
|
374
|
-
dtstart = get_date(arguments.pop("dtstart", None))
|
|
375
|
-
dtend = get_date(arguments.pop("dtend", None))
|
|
376
|
-
return get_date(dtstart), get_date(dtend)
|
|
377
|
-
|
|
378
|
-
|
|
379
|
-
def get_metadata_query_paths(metadata_mapping: Dict[str, Any]) -> Dict[str, Any]:
|
|
380
|
-
"""Get dict of query paths and their names from metadata_mapping
|
|
381
|
-
|
|
382
|
-
:param metadata_mapping: STAC metadata mapping (see 'resources/stac_provider.yml')
|
|
383
|
-
:type metadata_mapping: dict
|
|
384
|
-
:returns: Mapping of query paths with their corresponding names
|
|
385
|
-
:rtype: dict
|
|
386
|
-
"""
|
|
387
|
-
metadata_query_paths: Dict[str, Any] = {}
|
|
388
|
-
for metadata_name, metadata_spec in metadata_mapping.items():
|
|
389
|
-
# When metadata_spec have a length of 1 the query path is not specified
|
|
390
|
-
if len(metadata_spec) == 2:
|
|
391
|
-
metadata_query_template = metadata_spec[0]
|
|
392
|
-
try:
|
|
393
|
-
# We create the dict corresponding to the metadata query of the metadata
|
|
394
|
-
metadata_query_dict = ast.literal_eval(
|
|
395
|
-
metadata_query_template.format(**{metadata_name: None})
|
|
396
|
-
)
|
|
397
|
-
# We check if our query path pattern matches one or more of the dict path
|
|
398
|
-
matches = [
|
|
399
|
-
(str(match.full_path))
|
|
400
|
-
for match in string_to_jsonpath(
|
|
401
|
-
STAC_QUERY_PATTERN, force=True
|
|
402
|
-
).find(metadata_query_dict)
|
|
403
|
-
]
|
|
404
|
-
if matches:
|
|
405
|
-
metadata_query_path = matches[0]
|
|
406
|
-
metadata_query_paths[metadata_query_path] = metadata_name
|
|
407
|
-
except KeyError:
|
|
408
|
-
pass
|
|
409
|
-
return metadata_query_paths
|
|
410
|
-
|
|
411
|
-
|
|
412
|
-
def get_arguments_query_paths(arguments: Dict[str, Any]) -> Dict[str, Any]:
|
|
413
|
-
"""Get dict of query paths and their values from arguments
|
|
414
|
-
|
|
415
|
-
Build a mapping of the query paths present in the arguments
|
|
416
|
-
with their values. All matching paths of our STAC_QUERY_PATTERN
|
|
417
|
-
('query.*.*') are used.
|
|
418
|
-
|
|
419
|
-
:param arguments: Request args
|
|
420
|
-
:type arguments: dict
|
|
421
|
-
:returns: Mapping of query paths with their corresponding values
|
|
422
|
-
:rtype: dict
|
|
423
|
-
"""
|
|
424
|
-
return dict(
|
|
425
|
-
(str(match.full_path), match.value)
|
|
426
|
-
for match in string_to_jsonpath(STAC_QUERY_PATTERN, force=True).find(arguments)
|
|
427
|
-
)
|
|
428
|
-
|
|
429
|
-
|
|
430
|
-
def get_criterias_from_metadata_mapping(
|
|
431
|
-
metadata_mapping: Dict[str, Any], arguments: Dict[str, Any]
|
|
432
|
-
) -> Dict[str, Any]:
|
|
433
|
-
"""Get criterias from the search arguments with the metadata mapping config
|
|
434
|
-
|
|
435
|
-
:param metadata_mapping: STAC metadata mapping (see 'resources/stac_provider.yml')
|
|
436
|
-
:type metadata_mapping: dict
|
|
437
|
-
:param arguments: Request args
|
|
438
|
-
:type arguments: dict
|
|
439
|
-
:returns: Mapping of criterias with their corresponding values
|
|
440
|
-
:rtype: dict
|
|
441
|
-
"""
|
|
442
|
-
criterias: Dict[str, Any] = {}
|
|
443
|
-
metadata_query_paths = get_metadata_query_paths(metadata_mapping)
|
|
444
|
-
arguments_query_paths = get_arguments_query_paths(arguments)
|
|
445
|
-
for query_path in arguments_query_paths:
|
|
446
|
-
if query_path in metadata_query_paths:
|
|
447
|
-
criteria_name = metadata_query_paths[query_path]
|
|
448
|
-
else:
|
|
449
|
-
# The criteria is custom and we must read
|
|
450
|
-
# its name from the query path
|
|
451
|
-
criteria_name = query_path.split(".")[1]
|
|
452
|
-
criteria_value = arguments_query_paths[query_path]
|
|
453
|
-
criterias[criteria_name] = criteria_value
|
|
454
|
-
return criterias
|
|
455
|
-
|
|
456
|
-
|
|
457
|
-
def search_products(
|
|
458
|
-
product_type: str, arguments: Dict[str, Any], stac_formatted: bool = True
|
|
459
|
-
) -> Union[Dict[str, Any], SearchResult]:
|
|
460
|
-
"""Returns product search results
|
|
461
|
-
|
|
462
|
-
:param product_type: The product type criteria
|
|
463
|
-
:type product_type: str
|
|
464
|
-
:param arguments: Request args
|
|
465
|
-
:type arguments: dict
|
|
466
|
-
:param stac_formatted: Whether input is STAC-formatted or not
|
|
467
|
-
:type stac_formatted: bool
|
|
468
|
-
:returns: A search result
|
|
469
|
-
:rtype serialized GeoJSON response"""
|
|
470
|
-
|
|
471
|
-
try:
|
|
472
|
-
arg_product_type = arguments.pop("product_type", None)
|
|
473
|
-
provider = arguments.pop("provider", None)
|
|
474
|
-
|
|
475
|
-
unserialized = arguments.pop("unserialized", None)
|
|
476
|
-
|
|
477
|
-
page, items_per_page = get_pagination_info(arguments)
|
|
478
|
-
dtstart, dtend = get_datetime(arguments)
|
|
479
|
-
geom = get_geometry(arguments)
|
|
480
|
-
|
|
481
|
-
criterias = {
|
|
482
|
-
"productType": product_type if product_type else arg_product_type,
|
|
483
|
-
"page": page,
|
|
484
|
-
"items_per_page": items_per_page,
|
|
485
|
-
"start": dtstart,
|
|
486
|
-
"end": dtend,
|
|
487
|
-
"geom": geom,
|
|
488
|
-
"provider": provider,
|
|
489
|
-
}
|
|
490
|
-
|
|
491
|
-
if stac_formatted:
|
|
492
|
-
stac_provider_metadata_mapping = stac_provider_config.get("search", {}).get(
|
|
493
|
-
"metadata_mapping", {}
|
|
494
|
-
)
|
|
495
|
-
extra_criterias = get_criterias_from_metadata_mapping(
|
|
496
|
-
stac_provider_metadata_mapping, arguments
|
|
497
|
-
)
|
|
498
|
-
criterias.update(extra_criterias)
|
|
499
|
-
else:
|
|
500
|
-
criterias.update(arguments)
|
|
501
|
-
|
|
502
|
-
if provider:
|
|
503
|
-
criterias["raise_errors"] = True
|
|
504
|
-
|
|
505
|
-
# We remove potential None values to use the default values of the search method
|
|
506
|
-
criterias = dict((k, v) for k, v in criterias.items() if v is not None)
|
|
507
|
-
|
|
508
|
-
products, total = eodag_api.search(**criterias)
|
|
509
|
-
|
|
510
|
-
if not products and eodag_api.search_errors:
|
|
511
|
-
search_error = RequestError(
|
|
512
|
-
"No result could be obtained from any available provider and following "
|
|
513
|
-
"error(s) appeared while searching:"
|
|
514
|
-
)
|
|
515
|
-
search_error.history = eodag_api.search_errors
|
|
516
|
-
raise search_error
|
|
517
|
-
|
|
518
|
-
products = filter_products(products, arguments, **criterias)
|
|
519
|
-
|
|
520
|
-
response: Union[Dict[str, Any], SearchResult]
|
|
521
|
-
if not unserialized:
|
|
522
|
-
response = products.as_geojson_object()
|
|
523
|
-
response.update(
|
|
524
|
-
{
|
|
525
|
-
"properties": {
|
|
526
|
-
"page": page,
|
|
527
|
-
"itemsPerPage": items_per_page,
|
|
528
|
-
"totalResults": total,
|
|
529
|
-
}
|
|
530
|
-
}
|
|
531
|
-
)
|
|
532
|
-
else:
|
|
533
|
-
response = products
|
|
534
|
-
response.properties = {
|
|
535
|
-
"page": page,
|
|
536
|
-
"itemsPerPage": items_per_page,
|
|
537
|
-
"totalResults": total,
|
|
538
|
-
}
|
|
539
|
-
|
|
540
|
-
except ValidationError as e:
|
|
541
|
-
raise e
|
|
542
|
-
except RuntimeError as e:
|
|
543
|
-
raise e
|
|
544
|
-
except UnsupportedProductType as e:
|
|
545
|
-
raise e
|
|
546
|
-
|
|
547
|
-
return response
|
|
548
|
-
|
|
549
|
-
|
|
550
|
-
def search_product_by_id(
|
|
551
|
-
uid: str,
|
|
552
|
-
product_type: Optional[str] = None,
|
|
553
|
-
provider: Optional[str] = None,
|
|
554
|
-
**kwargs: Any,
|
|
555
|
-
) -> SearchResult:
|
|
556
|
-
"""Search a product by its id
|
|
557
|
-
|
|
558
|
-
:param uid: The uid of the EO product
|
|
559
|
-
:type uid: str
|
|
560
|
-
:param product_type: (optional) The product type
|
|
561
|
-
:type product_type: str
|
|
562
|
-
:param provider: (optional) The provider to be used
|
|
563
|
-
:type provider: str
|
|
564
|
-
:param kwargs: additional search parameters
|
|
565
|
-
:type kwargs: Any
|
|
566
|
-
:returns: A search result
|
|
567
|
-
:rtype: :class:`~eodag.api.search_result.SearchResult`
|
|
568
|
-
:raises: :class:`~eodag.utils.exceptions.ValidationError`
|
|
569
|
-
:raises: RuntimeError
|
|
570
|
-
"""
|
|
571
|
-
if provider:
|
|
572
|
-
kwargs["raise_errors"] = True
|
|
573
|
-
try:
|
|
574
|
-
products, _ = eodag_api.search(
|
|
575
|
-
id=uid, productType=product_type, provider=provider, **kwargs
|
|
576
|
-
)
|
|
577
|
-
return products
|
|
578
|
-
except ValidationError:
|
|
579
|
-
raise
|
|
580
|
-
except RuntimeError:
|
|
581
|
-
raise
|
|
582
|
-
|
|
583
|
-
|
|
584
|
-
# STAC ------------------------------------------------------------------------
|
|
585
|
-
|
|
586
|
-
|
|
587
|
-
def get_stac_conformance() -> Dict[str, str]:
|
|
588
|
-
"""Build STAC conformance
|
|
589
|
-
|
|
590
|
-
:returns: conformance dictionnary
|
|
591
|
-
:rtype: dict
|
|
592
|
-
"""
|
|
593
|
-
return stac_config["conformance"]
|
|
594
|
-
|
|
595
|
-
|
|
596
|
-
def get_stac_api_version() -> str:
|
|
597
|
-
"""Get STAC API version
|
|
598
|
-
|
|
599
|
-
:returns: STAC API version
|
|
600
|
-
:rtype: str
|
|
601
|
-
"""
|
|
602
|
-
return stac_config["stac_api_version"]
|
|
603
|
-
|
|
604
|
-
|
|
605
|
-
def get_stac_collections(
|
|
606
|
-
url: str, root: str, arguments: Dict[str, Any], provider: Optional[str] = None
|
|
607
|
-
) -> Dict[str, Any]:
|
|
608
|
-
"""Build STAC collections
|
|
609
|
-
|
|
610
|
-
:param url: Requested URL
|
|
611
|
-
:type url: str
|
|
612
|
-
:param root: The API root
|
|
613
|
-
:type root: str
|
|
614
|
-
:param arguments: Request args
|
|
615
|
-
:type arguments: dict
|
|
616
|
-
:param provider: (optional) Chosen provider
|
|
617
|
-
:type provider: str
|
|
618
|
-
:returns: Collections dictionnary
|
|
619
|
-
:rtype: dict
|
|
620
|
-
"""
|
|
621
|
-
return StacCollection(
|
|
622
|
-
url=url,
|
|
623
|
-
stac_config=stac_config,
|
|
624
|
-
provider=provider,
|
|
625
|
-
eodag_api=eodag_api,
|
|
626
|
-
root=root,
|
|
627
|
-
).get_collections(arguments)
|
|
628
|
-
|
|
629
|
-
|
|
630
|
-
def get_stac_collection_by_id(
|
|
631
|
-
url: str, root: str, collection_id: str, provider: Optional[str] = None
|
|
632
|
-
) -> Dict[str, Any]:
|
|
633
|
-
"""Build STAC collection by id
|
|
634
|
-
|
|
635
|
-
:param url: Requested URL
|
|
636
|
-
:type url: str
|
|
637
|
-
:param root: API root
|
|
638
|
-
:type root: str
|
|
639
|
-
:param collection_id: Product_type as ID of the collection
|
|
640
|
-
:type collection_id: str
|
|
641
|
-
:param provider: (optional) Chosen provider
|
|
642
|
-
:type provider: str
|
|
643
|
-
:returns: Collection dictionary
|
|
644
|
-
:rtype: dict
|
|
645
|
-
"""
|
|
646
|
-
return StacCollection(
|
|
647
|
-
url=url,
|
|
648
|
-
stac_config=stac_config,
|
|
649
|
-
provider=provider,
|
|
650
|
-
eodag_api=eodag_api,
|
|
651
|
-
root=root,
|
|
652
|
-
).get_collection_by_id(collection_id)
|
|
653
|
-
|
|
654
|
-
|
|
655
|
-
def get_stac_item_by_id(
|
|
656
|
-
url: str,
|
|
657
|
-
item_id: str,
|
|
658
|
-
catalogs: List[str],
|
|
659
|
-
root: str = "/",
|
|
660
|
-
provider: Optional[str] = None,
|
|
661
|
-
**kwargs: Any,
|
|
662
|
-
) -> Dict[str, Any]:
|
|
663
|
-
"""Build STAC item by id
|
|
664
|
-
|
|
665
|
-
:param url: Requested URL
|
|
666
|
-
:type url: str
|
|
667
|
-
:param item_id: Product ID
|
|
668
|
-
:type item_id: str
|
|
669
|
-
:param catalogs: Catalogs list (only first is used as product_type)
|
|
670
|
-
:type catalogs: list
|
|
671
|
-
:param root: (optional) API root
|
|
672
|
-
:type root: str
|
|
673
|
-
:param provider: (optional) Chosen provider
|
|
674
|
-
:type provider: str
|
|
675
|
-
:param kwargs: additional search parameters
|
|
676
|
-
:type kwargs: Any
|
|
677
|
-
:returns: Collection dictionary
|
|
678
|
-
:rtype: dict
|
|
679
|
-
"""
|
|
680
|
-
product_type = catalogs[0]
|
|
681
|
-
_dc_qs = kwargs.get("_dc_qs", None)
|
|
682
|
-
|
|
683
|
-
found_products = search_product_by_id(
|
|
684
|
-
item_id, product_type=product_type, provider=provider, _dc_qs=_dc_qs
|
|
685
|
-
)
|
|
686
|
-
|
|
687
|
-
if len(found_products) > 0:
|
|
688
|
-
found_products[0].product_type = eodag_api.get_alias_from_product_type(
|
|
689
|
-
found_products[0].product_type
|
|
690
|
-
)
|
|
691
|
-
return StacItem(
|
|
692
|
-
url=url,
|
|
693
|
-
stac_config=stac_config,
|
|
694
|
-
provider=provider,
|
|
695
|
-
eodag_api=eodag_api,
|
|
696
|
-
root=root,
|
|
697
|
-
).get_stac_item_from_product(product=found_products[0])
|
|
698
|
-
else:
|
|
699
|
-
return None
|
|
700
|
-
|
|
701
|
-
|
|
702
|
-
def download_stac_item_by_id_stream(
|
|
703
|
-
catalogs: List[str],
|
|
704
|
-
item_id: str,
|
|
705
|
-
provider: Optional[str] = None,
|
|
706
|
-
asset: Optional[str] = None,
|
|
707
|
-
**kwargs: Any,
|
|
708
|
-
) -> StreamingResponse:
|
|
709
|
-
"""Download item
|
|
710
|
-
|
|
711
|
-
:param catalogs: Catalogs list (only first is used as product_type)
|
|
712
|
-
:type catalogs: list
|
|
713
|
-
:param item_id: Product ID
|
|
714
|
-
:type item_id: str
|
|
715
|
-
:param provider: (optional) Chosen provider
|
|
716
|
-
:type provider: str
|
|
717
|
-
:param kwargs: additional download parameters
|
|
718
|
-
:type kwargs: Any
|
|
719
|
-
:returns: a stream of the downloaded data (zip file)
|
|
720
|
-
:rtype: StreamingResponse
|
|
721
|
-
"""
|
|
722
|
-
product_type = catalogs[0]
|
|
723
|
-
_dc_qs = kwargs.get("_dc_qs", None)
|
|
724
|
-
|
|
725
|
-
search_plugin = next(
|
|
726
|
-
eodag_api._plugins_manager.get_search_plugins(product_type, provider)
|
|
727
|
-
)
|
|
728
|
-
provider_product_type_config = search_plugin.config.products.get(
|
|
729
|
-
product_type, {}
|
|
730
|
-
) or search_plugin.config.products.get(GENERIC_PRODUCT_TYPE, {})
|
|
731
|
-
if provider_product_type_config.get("storeDownloadUrl", False):
|
|
732
|
-
if item_id not in search_plugin.download_info:
|
|
733
|
-
logger.error(f"data for item {item_id} not found")
|
|
734
|
-
raise NotAvailableError(
|
|
735
|
-
f"download url for product {item_id} could not be found, please redo "
|
|
736
|
-
f"the search request to fetch the required data"
|
|
737
|
-
)
|
|
738
|
-
product_data = search_plugin.download_info[item_id]
|
|
739
|
-
properties = {
|
|
740
|
-
"id": item_id,
|
|
741
|
-
"orderLink": product_data["orderLink"],
|
|
742
|
-
"downloadLink": product_data["downloadLink"],
|
|
743
|
-
"geometry": "-180 -90 180 90",
|
|
744
|
-
}
|
|
745
|
-
product = EOProduct(provider or product_data["provider"], properties)
|
|
746
|
-
else:
|
|
747
|
-
|
|
748
|
-
search_results = search_product_by_id(
|
|
749
|
-
item_id, product_type=product_type, provider=provider, _dc_qs=_dc_qs
|
|
750
|
-
)
|
|
751
|
-
if len(search_results) > 0:
|
|
752
|
-
product = search_results[0]
|
|
753
|
-
else:
|
|
754
|
-
raise NotAvailableError(
|
|
755
|
-
f"Could not find {item_id} item in {product_type} collection for provider {provider}"
|
|
756
|
-
)
|
|
757
|
-
|
|
758
|
-
if product.downloader is None:
|
|
759
|
-
download_plugin = eodag_api._plugins_manager.get_download_plugin(product)
|
|
760
|
-
auth_plugin = eodag_api._plugins_manager.get_auth_plugin(
|
|
761
|
-
download_plugin.provider
|
|
762
|
-
)
|
|
763
|
-
product.register_downloader(download_plugin, auth_plugin)
|
|
764
|
-
|
|
765
|
-
auth = (
|
|
766
|
-
product.downloader_auth.authenticate()
|
|
767
|
-
if product.downloader_auth is not None
|
|
768
|
-
else product.downloader_auth
|
|
769
|
-
)
|
|
770
|
-
try:
|
|
771
|
-
download_stream_dict = product.downloader._stream_download_dict(
|
|
772
|
-
product, auth=auth, asset=asset
|
|
773
|
-
)
|
|
774
|
-
except NotImplementedError:
|
|
775
|
-
logger.warning(
|
|
776
|
-
f"Download streaming not supported for {product.downloader}: downloading locally then delete"
|
|
777
|
-
)
|
|
778
|
-
product_path = eodag_api.download(product, extract=False, asset=asset)
|
|
779
|
-
if os.path.isdir(product_path):
|
|
780
|
-
# do not zip if dir contains only one file
|
|
781
|
-
all_filenames = [
|
|
782
|
-
f
|
|
783
|
-
for f in glob.glob(
|
|
784
|
-
os.path.join(product_path, "**", "*"), recursive=True
|
|
785
|
-
)
|
|
786
|
-
if os.path.isfile(f)
|
|
787
|
-
]
|
|
788
|
-
if len(all_filenames) == 1:
|
|
789
|
-
filepath_to_stream = all_filenames[0]
|
|
790
|
-
else:
|
|
791
|
-
filepath_to_stream = f"{product_path}.zip"
|
|
792
|
-
logger.debug(
|
|
793
|
-
f"Building archive for downloaded product path {filepath_to_stream}"
|
|
794
|
-
)
|
|
795
|
-
make_archive(product_path, "zip", product_path)
|
|
796
|
-
rmtree(product_path)
|
|
797
|
-
else:
|
|
798
|
-
filepath_to_stream = product_path
|
|
799
|
-
|
|
800
|
-
download_stream_dict = dict(
|
|
801
|
-
content=read_file_chunks_and_delete(open(filepath_to_stream, "rb")),
|
|
802
|
-
headers={
|
|
803
|
-
"content-disposition": f"attachment; filename={os.path.basename(filepath_to_stream)}",
|
|
804
|
-
},
|
|
805
|
-
)
|
|
806
|
-
|
|
807
|
-
return StreamingResponse(**download_stream_dict)
|
|
808
|
-
|
|
809
|
-
|
|
810
|
-
def read_file_chunks_and_delete(
|
|
811
|
-
opened_file: BufferedReader, chunk_size: int = 64 * 1024
|
|
812
|
-
) -> Iterator[bytes]:
|
|
813
|
-
"""Yield file chunks and delete file when finished."""
|
|
814
|
-
while True:
|
|
815
|
-
data = opened_file.read(chunk_size)
|
|
816
|
-
if not data:
|
|
817
|
-
opened_file.close()
|
|
818
|
-
os.remove(opened_file.name)
|
|
819
|
-
logger.debug(f"{opened_file.name} deleted after streaming complete")
|
|
820
|
-
break
|
|
821
|
-
yield data
|
|
822
|
-
yield data
|
|
823
|
-
|
|
824
|
-
|
|
825
|
-
def get_stac_catalogs(
|
|
826
|
-
url: str,
|
|
827
|
-
root: str = "/",
|
|
828
|
-
catalogs: List[str] = [],
|
|
829
|
-
provider: Optional[str] = None,
|
|
830
|
-
fetch_providers: bool = True,
|
|
831
|
-
) -> Dict[str, Any]:
|
|
832
|
-
"""Build STAC catalog
|
|
833
|
-
|
|
834
|
-
:param url: Requested URL
|
|
835
|
-
:type url: str
|
|
836
|
-
:param root: (optional) API root
|
|
837
|
-
:type root: str
|
|
838
|
-
:param catalogs: (optional) Catalogs list
|
|
839
|
-
:type catalogs: list
|
|
840
|
-
:param provider: (optional) Chosen provider
|
|
841
|
-
:type provider: str
|
|
842
|
-
:param fetch_providers: (optional) Whether to fetch providers for new product
|
|
843
|
-
types or not
|
|
844
|
-
:type fetch_providers: bool
|
|
845
|
-
:returns: Catalog dictionary
|
|
846
|
-
:rtype: dict
|
|
847
|
-
"""
|
|
848
|
-
return StacCatalog(
|
|
849
|
-
url=url,
|
|
850
|
-
stac_config=stac_config,
|
|
851
|
-
root=root,
|
|
852
|
-
provider=provider,
|
|
853
|
-
eodag_api=eodag_api,
|
|
854
|
-
catalogs=catalogs,
|
|
855
|
-
fetch_providers=fetch_providers,
|
|
856
|
-
).get_stac_catalog()
|
|
857
|
-
|
|
858
|
-
|
|
859
|
-
def search_stac_items(
|
|
860
|
-
url: str,
|
|
861
|
-
arguments: Dict[str, Any],
|
|
862
|
-
root: str = "/",
|
|
863
|
-
catalogs: List[str] = [],
|
|
864
|
-
provider: Optional[str] = None,
|
|
865
|
-
method: Optional[str] = "GET",
|
|
866
|
-
) -> Dict[str, Any]:
|
|
867
|
-
"""Get items collection dict for given catalogs list
|
|
868
|
-
|
|
869
|
-
:param url: Requested URL
|
|
870
|
-
:type url: str
|
|
871
|
-
:param arguments: Request args
|
|
872
|
-
:type arguments: dict
|
|
873
|
-
:param root: (optional) API root
|
|
874
|
-
:type root: str
|
|
875
|
-
:param catalogs: (optional) Catalogs list
|
|
876
|
-
:type catalogs: list
|
|
877
|
-
:param provider: (optional) Chosen provider
|
|
878
|
-
:type provider: str
|
|
879
|
-
:param method: (optional) search request HTTP method ('GET' or 'POST')
|
|
880
|
-
:type method: str
|
|
881
|
-
:returns: Catalog dictionnary
|
|
882
|
-
:rtype: dict
|
|
883
|
-
"""
|
|
884
|
-
collections = arguments.get("collections", None)
|
|
885
|
-
|
|
886
|
-
catalog_url = url.replace("/items", "")
|
|
887
|
-
|
|
888
|
-
next_page_kwargs = {
|
|
889
|
-
key: value for key, value in arguments.copy().items() if value is not None
|
|
890
|
-
}
|
|
891
|
-
next_page_id = (
|
|
892
|
-
int(next_page_kwargs["page"]) + 1 if "page" in next_page_kwargs else 2
|
|
893
|
-
)
|
|
894
|
-
next_page_kwargs["page"] = next_page_id
|
|
895
|
-
|
|
896
|
-
# use catalogs from path or if it is empty, collections from args
|
|
897
|
-
if catalogs:
|
|
898
|
-
result_catalog = StacCatalog(
|
|
899
|
-
url=catalog_url,
|
|
900
|
-
stac_config=stac_config,
|
|
901
|
-
root=root,
|
|
902
|
-
provider=provider,
|
|
903
|
-
eodag_api=eodag_api,
|
|
904
|
-
catalogs=catalogs,
|
|
905
|
-
)
|
|
906
|
-
elif collections:
|
|
907
|
-
# get collection as product_type
|
|
908
|
-
if isinstance(collections, str):
|
|
909
|
-
collections = collections.split(",")
|
|
910
|
-
elif not isinstance(collections, list):
|
|
911
|
-
raise ValidationError("Collections argument type should be Array")
|
|
912
|
-
|
|
913
|
-
result_catalog = StacCatalog(
|
|
914
|
-
stac_config=stac_config,
|
|
915
|
-
root=root,
|
|
916
|
-
provider=provider,
|
|
917
|
-
eodag_api=eodag_api,
|
|
918
|
-
# handle only one collection
|
|
919
|
-
# per request (STAC allows multiple)
|
|
920
|
-
catalogs=collections[0:1],
|
|
921
|
-
url=catalog_url.replace("/search", f"/collections/{collections[0]}"),
|
|
922
|
-
)
|
|
923
|
-
arguments.pop("collections")
|
|
924
|
-
else:
|
|
925
|
-
raise NoMatchingProductType("Invalid request, collections argument is missing")
|
|
926
|
-
|
|
927
|
-
# get products by ids
|
|
928
|
-
ids = arguments.get("ids", None)
|
|
929
|
-
if isinstance(ids, str):
|
|
930
|
-
ids = [ids]
|
|
931
|
-
if ids:
|
|
932
|
-
search_results = SearchResult([])
|
|
933
|
-
for item_id in ids:
|
|
934
|
-
found_products = search_product_by_id(
|
|
935
|
-
item_id, product_type=collections[0], provider=provider
|
|
936
|
-
)
|
|
937
|
-
if len(found_products) == 1:
|
|
938
|
-
search_results.extend(found_products)
|
|
939
|
-
search_results.properties = {
|
|
940
|
-
"page": 1,
|
|
941
|
-
"itemsPerPage": len(search_results),
|
|
942
|
-
"totalResults": len(search_results),
|
|
943
|
-
}
|
|
944
|
-
else:
|
|
945
|
-
if "datetime" in arguments.keys() and arguments["datetime"] is not None:
|
|
946
|
-
arguments["dtstart"], arguments["dtend"] = get_datetime(arguments)
|
|
947
|
-
|
|
948
|
-
search_products_arguments = dict(
|
|
949
|
-
arguments,
|
|
950
|
-
**result_catalog.search_args,
|
|
951
|
-
**{"unserialized": "true", "provider": provider},
|
|
952
|
-
)
|
|
953
|
-
|
|
954
|
-
# check if time filtering appears both in search arguments and catalog
|
|
955
|
-
# (for catalogs built by date: i.e. `year/2020/month/05`)
|
|
956
|
-
if set(["dtstart", "dtend"]) <= set(arguments.keys()) and set(
|
|
957
|
-
["dtstart", "dtend"]
|
|
958
|
-
) <= set(result_catalog.search_args.keys()):
|
|
959
|
-
search_date_min = (
|
|
960
|
-
dateutil.parser.parse(arguments["dtstart"])
|
|
961
|
-
if arguments["dtstart"]
|
|
962
|
-
else datetime.datetime.min
|
|
963
|
-
)
|
|
964
|
-
search_date_max = (
|
|
965
|
-
dateutil.parser.parse(arguments["dtend"])
|
|
966
|
-
if arguments["dtend"]
|
|
967
|
-
else datetime.datetime.now()
|
|
968
|
-
)
|
|
969
|
-
catalog_date_min = dateutil.parser.parse(
|
|
970
|
-
result_catalog.search_args["dtstart"]
|
|
971
|
-
)
|
|
972
|
-
catalog_date_max = dateutil.parser.parse(
|
|
973
|
-
result_catalog.search_args["dtend"]
|
|
974
|
-
)
|
|
975
|
-
# check if date intervals overlap
|
|
976
|
-
if (search_date_min <= catalog_date_max) and (
|
|
977
|
-
search_date_max >= catalog_date_min
|
|
978
|
-
):
|
|
979
|
-
# use intersection
|
|
980
|
-
search_products_arguments["dtstart"] = (
|
|
981
|
-
max(search_date_min, catalog_date_min)
|
|
982
|
-
.isoformat()
|
|
983
|
-
.replace("+00:00", "")
|
|
984
|
-
+ "Z"
|
|
985
|
-
)
|
|
986
|
-
search_products_arguments["dtend"] = (
|
|
987
|
-
min(search_date_max, catalog_date_max)
|
|
988
|
-
.isoformat()
|
|
989
|
-
.replace("+00:00", "")
|
|
990
|
-
+ "Z"
|
|
991
|
-
)
|
|
992
|
-
else:
|
|
993
|
-
logger.warning("Time intervals do not overlap")
|
|
994
|
-
# return empty results
|
|
995
|
-
search_results = SearchResult([])
|
|
996
|
-
search_results.properties = {
|
|
997
|
-
"page": search_products_arguments.get("page", 1),
|
|
998
|
-
"itemsPerPage": search_products_arguments.get(
|
|
999
|
-
"itemsPerPage", DEFAULT_ITEMS_PER_PAGE
|
|
1000
|
-
),
|
|
1001
|
-
"totalResults": 0,
|
|
1002
|
-
}
|
|
1003
|
-
return StacItem(
|
|
1004
|
-
url=url,
|
|
1005
|
-
stac_config=stac_config,
|
|
1006
|
-
provider=provider,
|
|
1007
|
-
eodag_api=eodag_api,
|
|
1008
|
-
root=root,
|
|
1009
|
-
).get_stac_items(
|
|
1010
|
-
search_results=search_results,
|
|
1011
|
-
catalog=dict(
|
|
1012
|
-
result_catalog.get_stac_catalog(),
|
|
1013
|
-
**{"url": result_catalog.url, "root": result_catalog.root},
|
|
1014
|
-
),
|
|
1015
|
-
)
|
|
1016
|
-
|
|
1017
|
-
search_results = search_products(
|
|
1018
|
-
product_type=result_catalog.search_args["product_type"],
|
|
1019
|
-
arguments=search_products_arguments,
|
|
1020
|
-
)
|
|
1021
|
-
|
|
1022
|
-
for record in search_results:
|
|
1023
|
-
record.product_type = eodag_api.get_alias_from_product_type(record.product_type)
|
|
1024
|
-
|
|
1025
|
-
search_results.method = method
|
|
1026
|
-
if method == "POST":
|
|
1027
|
-
search_results.next = f"{url}"
|
|
1028
|
-
search_results.body = next_page_kwargs
|
|
1029
|
-
|
|
1030
|
-
elif method == "GET":
|
|
1031
|
-
next_query_string = urlencode(next_page_kwargs)
|
|
1032
|
-
search_results.next = f"{url}?{next_query_string}"
|
|
1033
|
-
|
|
1034
|
-
items = StacItem(
|
|
1035
|
-
url=url,
|
|
1036
|
-
stac_config=stac_config,
|
|
1037
|
-
provider=provider,
|
|
1038
|
-
eodag_api=eodag_api,
|
|
1039
|
-
root=root,
|
|
1040
|
-
).get_stac_items(
|
|
1041
|
-
search_results=search_results,
|
|
1042
|
-
catalog=dict(
|
|
1043
|
-
result_catalog.get_stac_catalog(),
|
|
1044
|
-
**{"url": result_catalog.url, "root": result_catalog.root},
|
|
1045
|
-
),
|
|
1046
|
-
)
|
|
1047
|
-
|
|
1048
|
-
return items
|
|
1049
|
-
|
|
1050
|
-
|
|
1051
|
-
def get_stac_extension_oseo(url: str) -> Dict[str, str]:
|
|
1052
|
-
"""Build STAC OGC / OpenSearch Extension for EO
|
|
1053
|
-
|
|
1054
|
-
:param url: Requested URL
|
|
1055
|
-
:type url: str
|
|
1056
|
-
:returns: Catalog dictionnary
|
|
1057
|
-
:rtype: dict
|
|
1058
|
-
"""
|
|
1059
|
-
|
|
1060
|
-
item_mapping = dict_items_recursive_apply(
|
|
1061
|
-
stac_config["item"], lambda _, x: str(x).replace("$.product.", "$.")
|
|
1062
|
-
)
|
|
1063
|
-
|
|
1064
|
-
# all properties as string type by default
|
|
1065
|
-
oseo_properties = {
|
|
1066
|
-
"oseo:{}".format(k): {
|
|
1067
|
-
"type": "string",
|
|
1068
|
-
"title": k[0].upper() + re.sub(r"([A-Z][a-z]+)", r" \1", k[1:]),
|
|
1069
|
-
}
|
|
1070
|
-
for k, v in OSEO_METADATA_MAPPING.items()
|
|
1071
|
-
if v not in str(item_mapping)
|
|
1072
|
-
}
|
|
1073
|
-
|
|
1074
|
-
return StacCommon.get_stac_extension(
|
|
1075
|
-
url=url, stac_config=stac_config, extension="oseo", properties=oseo_properties
|
|
1076
|
-
)
|
|
1077
|
-
|
|
1078
|
-
|
|
1079
|
-
def fetch_collection_queryable_properties(
|
|
1080
|
-
collection_id: Optional[str] = None, provider: Optional[str] = None, **kwargs: Any
|
|
1081
|
-
) -> Dict[str, StacQueryableProperty]:
|
|
1082
|
-
"""Fetch the queryable properties for a collection.
|
|
1083
|
-
|
|
1084
|
-
:param collection_id: The ID of the collection.
|
|
1085
|
-
:type collection_id: str
|
|
1086
|
-
:param provider: (optional) The provider.
|
|
1087
|
-
:type provider: str
|
|
1088
|
-
:param kwargs: additional filters for queryables (`productType` or other search
|
|
1089
|
-
arguments)
|
|
1090
|
-
:type kwargs: Any
|
|
1091
|
-
:returns: A set containing the STAC standardized queryable properties for a collection.
|
|
1092
|
-
:rtype Dict[str, StacQueryableProperty]: set
|
|
1093
|
-
"""
|
|
1094
|
-
if not collection_id and "collections" in kwargs:
|
|
1095
|
-
collection_ids = kwargs.pop("collections").split(",")
|
|
1096
|
-
collection_id = collection_ids[0]
|
|
1097
|
-
|
|
1098
|
-
if collection_id and "productType" in kwargs:
|
|
1099
|
-
kwargs.pop("productType")
|
|
1100
|
-
elif "productType" in kwargs:
|
|
1101
|
-
collection_id = kwargs.pop("productType")
|
|
1102
|
-
|
|
1103
|
-
if "ids" in kwargs:
|
|
1104
|
-
kwargs["id"] = kwargs.pop("ids")
|
|
1105
|
-
|
|
1106
|
-
if "datetime" in kwargs:
|
|
1107
|
-
dates = get_datetime(kwargs)
|
|
1108
|
-
kwargs["start"] = dates[0]
|
|
1109
|
-
kwargs["end"] = dates[1]
|
|
1110
|
-
|
|
1111
|
-
python_queryables = eodag_api.list_queryables(
|
|
1112
|
-
provider=provider, productType=collection_id, **kwargs
|
|
1113
|
-
)
|
|
1114
|
-
python_queryables.pop("start")
|
|
1115
|
-
python_queryables.pop("end")
|
|
1116
|
-
|
|
1117
|
-
stac_queryables = dict()
|
|
1118
|
-
for param, queryable in python_queryables.items():
|
|
1119
|
-
stac_param = EODAGSearch.to_stac(param)
|
|
1120
|
-
stac_queryables[
|
|
1121
|
-
stac_param
|
|
1122
|
-
] = StacQueryableProperty.from_python_field_definition(stac_param, queryable)
|
|
1123
|
-
|
|
1124
|
-
return stac_queryables
|
|
1125
|
-
|
|
1126
|
-
|
|
1127
|
-
def eodag_api_init() -> None:
|
|
1128
|
-
"""Init EODataAccessGateway server instance, pre-running all time consuming tasks"""
|
|
1129
|
-
eodag_api.fetch_product_types_list()
|
|
1130
|
-
|
|
1131
|
-
# pre-build search plugins
|
|
1132
|
-
for provider in eodag_api.available_providers():
|
|
1133
|
-
next(eodag_api._plugins_manager.get_search_plugins(provider=provider))
|