eodag 2.12.1__py3-none-any.whl → 3.0.0b1__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (77) hide show
  1. eodag/api/core.py +434 -319
  2. eodag/api/product/__init__.py +5 -1
  3. eodag/api/product/_assets.py +7 -2
  4. eodag/api/product/_product.py +46 -68
  5. eodag/api/product/metadata_mapping.py +181 -66
  6. eodag/api/search_result.py +21 -1
  7. eodag/cli.py +20 -6
  8. eodag/config.py +95 -6
  9. eodag/plugins/apis/base.py +8 -165
  10. eodag/plugins/apis/ecmwf.py +36 -24
  11. eodag/plugins/apis/usgs.py +40 -24
  12. eodag/plugins/authentication/aws_auth.py +2 -2
  13. eodag/plugins/authentication/header.py +31 -6
  14. eodag/plugins/authentication/keycloak.py +13 -84
  15. eodag/plugins/authentication/oauth.py +3 -3
  16. eodag/plugins/authentication/openid_connect.py +256 -46
  17. eodag/plugins/authentication/qsauth.py +3 -0
  18. eodag/plugins/authentication/sas_auth.py +8 -1
  19. eodag/plugins/authentication/token.py +92 -46
  20. eodag/plugins/authentication/token_exchange.py +120 -0
  21. eodag/plugins/download/aws.py +86 -91
  22. eodag/plugins/download/base.py +72 -40
  23. eodag/plugins/download/http.py +607 -264
  24. eodag/plugins/download/s3rest.py +28 -15
  25. eodag/plugins/manager.py +73 -57
  26. eodag/plugins/search/__init__.py +36 -0
  27. eodag/plugins/search/base.py +225 -18
  28. eodag/plugins/search/build_search_result.py +389 -32
  29. eodag/plugins/search/cop_marine.py +378 -0
  30. eodag/plugins/search/creodias_s3.py +15 -14
  31. eodag/plugins/search/csw.py +5 -7
  32. eodag/plugins/search/data_request_search.py +44 -20
  33. eodag/plugins/search/qssearch.py +508 -203
  34. eodag/plugins/search/static_stac_search.py +99 -36
  35. eodag/resources/constraints/climate-dt.json +13 -0
  36. eodag/resources/constraints/extremes-dt.json +8 -0
  37. eodag/resources/ext_product_types.json +1 -1
  38. eodag/resources/product_types.yml +1897 -34
  39. eodag/resources/providers.yml +3539 -3277
  40. eodag/resources/stac.yml +48 -54
  41. eodag/resources/stac_api.yml +71 -25
  42. eodag/resources/stac_provider.yml +5 -0
  43. eodag/resources/user_conf_template.yml +51 -3
  44. eodag/rest/__init__.py +6 -0
  45. eodag/rest/cache.py +70 -0
  46. eodag/rest/config.py +68 -0
  47. eodag/rest/constants.py +27 -0
  48. eodag/rest/core.py +757 -0
  49. eodag/rest/server.py +397 -258
  50. eodag/rest/stac.py +438 -307
  51. eodag/rest/types/collections_search.py +44 -0
  52. eodag/rest/types/eodag_search.py +232 -43
  53. eodag/rest/types/{stac_queryables.py → queryables.py} +81 -43
  54. eodag/rest/types/stac_search.py +277 -0
  55. eodag/rest/utils/__init__.py +216 -0
  56. eodag/rest/utils/cql_evaluate.py +119 -0
  57. eodag/rest/utils/rfc3339.py +65 -0
  58. eodag/types/__init__.py +99 -9
  59. eodag/types/bbox.py +15 -14
  60. eodag/types/download_args.py +31 -0
  61. eodag/types/search_args.py +58 -7
  62. eodag/types/whoosh.py +81 -0
  63. eodag/utils/__init__.py +72 -9
  64. eodag/utils/constraints.py +37 -37
  65. eodag/utils/exceptions.py +23 -17
  66. eodag/utils/requests.py +138 -0
  67. eodag/utils/rest.py +104 -0
  68. eodag/utils/stac_reader.py +100 -16
  69. {eodag-2.12.1.dist-info → eodag-3.0.0b1.dist-info}/METADATA +64 -44
  70. eodag-3.0.0b1.dist-info/RECORD +109 -0
  71. {eodag-2.12.1.dist-info → eodag-3.0.0b1.dist-info}/WHEEL +1 -1
  72. {eodag-2.12.1.dist-info → eodag-3.0.0b1.dist-info}/entry_points.txt +6 -5
  73. eodag/plugins/apis/cds.py +0 -540
  74. eodag/rest/utils.py +0 -1133
  75. eodag-2.12.1.dist-info/RECORD +0 -94
  76. {eodag-2.12.1.dist-info → eodag-3.0.0b1.dist-info}/LICENSE +0 -0
  77. {eodag-2.12.1.dist-info → eodag-3.0.0b1.dist-info}/top_level.txt +0 -0
eodag/rest/utils.py DELETED
@@ -1,1133 +0,0 @@
1
- # -*- coding: utf-8 -*-
2
- # Copyright 2018, CS Systemes d'Information, https://www.csgroup.eu/
3
- #
4
- # This file is part of EODAG project
5
- # https://www.github.com/CS-SI/EODAG
6
- #
7
- # Licensed under the Apache License, Version 2.0 (the "License");
8
- # you may not use this file except in compliance with the License.
9
- # You may obtain a copy of the License at
10
- #
11
- # http://www.apache.org/licenses/LICENSE-2.0
12
- #
13
- # Unless required by applicable law or agreed to in writing, software
14
- # distributed under the License is distributed on an "AS IS" BASIS,
15
- # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16
- # See the License for the specific language governing permissions and
17
- # limitations under the License.
18
- from __future__ import annotations
19
-
20
- import ast
21
- import datetime
22
- import glob
23
- import json
24
- import logging
25
- import os
26
- import re
27
- from shutil import make_archive, rmtree
28
- from typing import (
29
- TYPE_CHECKING,
30
- Any,
31
- Callable,
32
- Dict,
33
- Iterator,
34
- List,
35
- NamedTuple,
36
- Optional,
37
- Tuple,
38
- Union,
39
- )
40
- from urllib.parse import urlencode
41
-
42
- import dateutil.parser
43
- from dateutil import tz
44
- from fastapi.responses import StreamingResponse
45
- from shapely.geometry import Polygon, shape
46
-
47
- import eodag
48
- from eodag import EOProduct
49
- from eodag.api.product.metadata_mapping import OSEO_METADATA_MAPPING
50
- from eodag.api.search_result import SearchResult
51
- from eodag.config import load_stac_config, load_stac_provider_config
52
- from eodag.plugins.crunch.filter_latest_intersect import FilterLatestIntersect
53
- from eodag.plugins.crunch.filter_latest_tpl_name import FilterLatestByName
54
- from eodag.plugins.crunch.filter_overlap import FilterOverlap
55
- from eodag.rest.stac import StacCatalog, StacCollection, StacCommon, StacItem
56
- from eodag.rest.types.eodag_search import EODAGSearch
57
- from eodag.rest.types.stac_queryables import StacQueryableProperty
58
- from eodag.utils import (
59
- DEFAULT_ITEMS_PER_PAGE,
60
- DEFAULT_PAGE,
61
- GENERIC_PRODUCT_TYPE,
62
- _deprecated,
63
- dict_items_recursive_apply,
64
- string_to_jsonpath,
65
- )
66
- from eodag.utils.exceptions import (
67
- MisconfiguredError,
68
- NoMatchingProductType,
69
- NotAvailableError,
70
- RequestError,
71
- UnsupportedProductType,
72
- ValidationError,
73
- )
74
-
75
- if TYPE_CHECKING:
76
- from io import BufferedReader
77
-
78
- from shapely.geometry.base import BaseGeometry
79
-
80
-
81
- logger = logging.getLogger("eodag.rest.utils")
82
-
83
- eodag_api = eodag.EODataAccessGateway()
84
-
85
-
86
- class Cruncher(NamedTuple):
87
- """Type hinted Cruncher namedTuple"""
88
-
89
- clazz: Callable[..., Any]
90
- config_params: List[str]
91
-
92
-
93
- crunchers = {
94
- "latestIntersect": Cruncher(FilterLatestIntersect, []),
95
- "latestByName": Cruncher(FilterLatestByName, ["name_pattern"]),
96
- "overlap": Cruncher(FilterOverlap, ["minimum_overlap"]),
97
- }
98
- stac_config = load_stac_config()
99
- stac_provider_config = load_stac_provider_config()
100
-
101
- STAC_QUERY_PATTERN = "query.*.*"
102
-
103
-
104
- @_deprecated(
105
- reason="Function internally used by get_home_page_content, also deprecated",
106
- version="2.6.1",
107
- )
108
- def format_product_types(product_types: List[Dict[str, Any]]) -> str:
109
- """Format product_types
110
-
111
- :param product_types: A list of EODAG product types as returned by the core api
112
- :type product_types: list
113
- """
114
- result: List[str] = []
115
- for pt in product_types:
116
- result.append("* *__{ID}__*: {abstract}".format(**pt))
117
- return "\n".join(sorted(result))
118
-
119
-
120
- def get_detailled_collections_list(
121
- provider: Optional[str] = None, fetch_providers: bool = True
122
- ) -> List[Dict[str, Any]]:
123
- """Returns detailled collections / product_types list for a given provider as a list of config dicts
124
-
125
- :param provider: (optional) Chosen provider
126
- :type provider: str
127
- :param fetch_providers: (optional) Whether to fetch providers for new product
128
- types or not
129
- :type fetch_providers: bool
130
- :returns: List of config dicts
131
- :rtype: list
132
- """
133
- return eodag_api.list_product_types(
134
- provider=provider, fetch_providers=fetch_providers
135
- )
136
-
137
-
138
- @_deprecated(reason="No more needed with STAC API + Swagger", version="2.6.1")
139
- def get_home_page_content(base_url: str, ipp: Optional[int] = None) -> str:
140
- """Compute eodag service home page content
141
-
142
- :param base_url: The service root URL
143
- :type base_url: str
144
- :param ipp: (optional) Items per page number
145
- :type ipp: int
146
- """
147
- base_url = base_url.rstrip("/") + "/"
148
- content = f"""<h1>EODAG Server</h1><br />
149
- <a href='{base_url}'>root</a><br />
150
- <a href='{base_url}service-doc'>service-doc</a><br />
151
- """
152
- return content
153
-
154
-
155
- @_deprecated(
156
- reason="Used to format output from deprecated function get_home_page_content",
157
- version="2.6.1",
158
- )
159
- def get_templates_path() -> str:
160
- """Returns Jinja templates path"""
161
- return os.path.join(os.path.dirname(__file__), "templates")
162
-
163
-
164
- def get_product_types(
165
- provider: Optional[str] = None, filters: Optional[Dict[str, Any]] = None
166
- ) -> List[Dict[str, Any]]:
167
- """Returns a list of supported product types
168
-
169
- :param provider: (optional) Provider name
170
- :type provider: str
171
- :param filters: (optional) Additional filters for product types search
172
- :type filters: dict
173
- :returns: A list of corresponding product types
174
- :rtype: list
175
- """
176
- if filters is None:
177
- filters = {}
178
- try:
179
- guessed_product_types = eodag_api.guess_product_type(
180
- instrument=filters.get("instrument"),
181
- platform=filters.get("platform"),
182
- platformSerialIdentifier=filters.get("platformSerialIdentifier"),
183
- sensorType=filters.get("sensorType"),
184
- processingLevel=filters.get("processingLevel"),
185
- )
186
- except NoMatchingProductType:
187
- guessed_product_types = []
188
- if guessed_product_types:
189
- product_types = [
190
- pt
191
- for pt in eodag_api.list_product_types(provider=provider)
192
- if pt["ID"] in guessed_product_types
193
- ]
194
- else:
195
- product_types = eodag_api.list_product_types(provider=provider)
196
- return product_types
197
-
198
-
199
- def search_bbox(request_bbox: str) -> Optional[Dict[str, float]]:
200
- """Transform request bounding box as a bbox suitable for eodag search"""
201
-
202
- eodag_bbox = None
203
- search_bbox_keys = ["lonmin", "latmin", "lonmax", "latmax"]
204
-
205
- if not request_bbox:
206
- return None
207
-
208
- try:
209
- request_bbox_list = [float(coord) for coord in request_bbox.split(",")]
210
- except ValueError as e:
211
- raise ValidationError("invalid box coordinate type: %s" % e)
212
-
213
- eodag_bbox = dict(zip(search_bbox_keys, request_bbox_list))
214
- if len(eodag_bbox) != 4:
215
- raise ValidationError("input box is invalid: %s" % request_bbox)
216
-
217
- return eodag_bbox
218
-
219
-
220
- def get_date(date: Optional[str]) -> Optional[str]:
221
- """Check if the input date can be parsed as a date"""
222
-
223
- if not date:
224
- return None
225
- try:
226
- return (
227
- dateutil.parser.parse(date)
228
- .replace(tzinfo=tz.UTC)
229
- .isoformat()
230
- .replace("+00:00", "")
231
- )
232
- except ValueError as e:
233
- exc = ValidationError("invalid input date: %s" % e)
234
- raise exc
235
-
236
-
237
- def get_int(input: Optional[Any]) -> Optional[int]:
238
- """Check if the input can be parsed as an integer"""
239
-
240
- if input is None:
241
- return None
242
-
243
- try:
244
- val = int(input)
245
- except ValueError as e:
246
- raise ValidationError("invalid input integer value: %s" % e)
247
- return val
248
-
249
-
250
- def filter_products(
251
- products: SearchResult, arguments: Dict[str, Any], **kwargs: Any
252
- ) -> SearchResult:
253
- """Apply an eodag cruncher to filter products"""
254
- filter_name = arguments.get("filter")
255
- if filter_name:
256
- cruncher = crunchers.get(filter_name)
257
- if not cruncher:
258
- raise ValidationError("unknown filter name")
259
-
260
- cruncher_config: Dict[str, Any] = dict()
261
- for config_param in cruncher.config_params:
262
- config_param_value = arguments.get(config_param)
263
- if not config_param_value:
264
- raise ValidationError(
265
- "filter additional parameters required: %s"
266
- % ", ".join(cruncher.config_params)
267
- )
268
- cruncher_config[config_param] = config_param_value
269
-
270
- try:
271
- products = products.crunch(cruncher.clazz(cruncher_config), **kwargs)
272
- except MisconfiguredError as e:
273
- raise ValidationError(str(e))
274
-
275
- return products
276
-
277
-
278
- def get_pagination_info(
279
- arguments: Dict[str, Any]
280
- ) -> Tuple[Optional[int], Optional[int]]:
281
- """Get pagination arguments"""
282
- page = get_int(arguments.pop("page", DEFAULT_PAGE))
283
- # items_per_page can be specified using limit or itemsPerPage
284
- items_per_page = get_int(arguments.pop("limit", DEFAULT_ITEMS_PER_PAGE))
285
- items_per_page = get_int(arguments.pop("itemsPerPage", items_per_page))
286
-
287
- if page is not None and page < 0:
288
- raise ValidationError("invalid page number. Must be positive integer")
289
- if items_per_page is not None and items_per_page < 0:
290
- raise ValidationError(
291
- "invalid number of items per page. Must be positive integer"
292
- )
293
- return page, items_per_page
294
-
295
-
296
- def get_geometry(arguments: Dict[str, Any]) -> Optional[BaseGeometry]:
297
- """Get geometry from arguments"""
298
- if arguments.get("intersects") and arguments.get("bbox"):
299
- raise ValidationError("Only one of bbox and intersects can be used at a time.")
300
-
301
- if arguments.get("bbox"):
302
- request_bbox = arguments.pop("bbox")
303
- if isinstance(request_bbox, str):
304
- request_bbox = request_bbox.split(",")
305
- elif not isinstance(request_bbox, list):
306
- raise ValidationError("bbox argument type should be Array")
307
-
308
- try:
309
- request_bbox = [float(coord) for coord in request_bbox]
310
- except ValueError as e:
311
- raise ValidationError(f"invalid bbox coordinate type: {e}")
312
-
313
- if len(request_bbox) == 4:
314
- min_x, min_y, max_x, max_y = request_bbox
315
- elif len(request_bbox) == 6:
316
- min_x, min_y, _, max_x, max_y, _ = request_bbox
317
- else:
318
- raise ValidationError(
319
- f"invalid bbox length ({len(request_bbox)}) for bbox {request_bbox}"
320
- )
321
-
322
- geom = Polygon([(min_x, min_y), (min_x, max_y), (max_x, max_y), (max_x, min_y)])
323
-
324
- elif arguments.get("intersects"):
325
- intersects_value = arguments.pop("intersects")
326
- if isinstance(intersects_value, str):
327
- try:
328
- intersects_dict = json.loads(intersects_value)
329
- except json.JSONDecodeError:
330
- raise ValidationError(
331
- "The 'intersects' parameter is not a valid JSON string."
332
- )
333
- else:
334
- intersects_dict = intersects_value
335
-
336
- try:
337
- geom = shape(intersects_dict)
338
- except Exception as e:
339
- raise ValidationError(
340
- f"The 'intersects' parameter does not represent a valid geometry: {str(e)}"
341
- )
342
-
343
- else:
344
- geom = None
345
-
346
- return geom
347
-
348
-
349
- def get_datetime(arguments: Dict[str, Any]) -> Tuple[Optional[str], Optional[str]]:
350
- """Get the datetime criterias from the search arguments
351
-
352
- :param arguments: Request args
353
- :type arguments: dict
354
- :returns: Start date and end date from datetime string.
355
- :rtype: Tuple[Optional[str], Optional[str]]
356
- """
357
- datetime_str = arguments.pop("datetime", None)
358
-
359
- if datetime_str:
360
- datetime_split = datetime_str.split("/")
361
- if len(datetime_split) > 1:
362
- dtstart = datetime_split[0] if datetime_split[0] != ".." else None
363
- dtend = datetime_split[1] if datetime_split[1] != ".." else None
364
- elif len(datetime_split) == 1:
365
- # same time for start & end if only one is given
366
- dtstart, dtend = datetime_split[0:1] * 2
367
- else:
368
- return None, None
369
-
370
- return get_date(dtstart), get_date(dtend)
371
-
372
- else:
373
- # return already set (dtstart, dtend) or None
374
- dtstart = get_date(arguments.pop("dtstart", None))
375
- dtend = get_date(arguments.pop("dtend", None))
376
- return get_date(dtstart), get_date(dtend)
377
-
378
-
379
- def get_metadata_query_paths(metadata_mapping: Dict[str, Any]) -> Dict[str, Any]:
380
- """Get dict of query paths and their names from metadata_mapping
381
-
382
- :param metadata_mapping: STAC metadata mapping (see 'resources/stac_provider.yml')
383
- :type metadata_mapping: dict
384
- :returns: Mapping of query paths with their corresponding names
385
- :rtype: dict
386
- """
387
- metadata_query_paths: Dict[str, Any] = {}
388
- for metadata_name, metadata_spec in metadata_mapping.items():
389
- # When metadata_spec have a length of 1 the query path is not specified
390
- if len(metadata_spec) == 2:
391
- metadata_query_template = metadata_spec[0]
392
- try:
393
- # We create the dict corresponding to the metadata query of the metadata
394
- metadata_query_dict = ast.literal_eval(
395
- metadata_query_template.format(**{metadata_name: None})
396
- )
397
- # We check if our query path pattern matches one or more of the dict path
398
- matches = [
399
- (str(match.full_path))
400
- for match in string_to_jsonpath(
401
- STAC_QUERY_PATTERN, force=True
402
- ).find(metadata_query_dict)
403
- ]
404
- if matches:
405
- metadata_query_path = matches[0]
406
- metadata_query_paths[metadata_query_path] = metadata_name
407
- except KeyError:
408
- pass
409
- return metadata_query_paths
410
-
411
-
412
- def get_arguments_query_paths(arguments: Dict[str, Any]) -> Dict[str, Any]:
413
- """Get dict of query paths and their values from arguments
414
-
415
- Build a mapping of the query paths present in the arguments
416
- with their values. All matching paths of our STAC_QUERY_PATTERN
417
- ('query.*.*') are used.
418
-
419
- :param arguments: Request args
420
- :type arguments: dict
421
- :returns: Mapping of query paths with their corresponding values
422
- :rtype: dict
423
- """
424
- return dict(
425
- (str(match.full_path), match.value)
426
- for match in string_to_jsonpath(STAC_QUERY_PATTERN, force=True).find(arguments)
427
- )
428
-
429
-
430
- def get_criterias_from_metadata_mapping(
431
- metadata_mapping: Dict[str, Any], arguments: Dict[str, Any]
432
- ) -> Dict[str, Any]:
433
- """Get criterias from the search arguments with the metadata mapping config
434
-
435
- :param metadata_mapping: STAC metadata mapping (see 'resources/stac_provider.yml')
436
- :type metadata_mapping: dict
437
- :param arguments: Request args
438
- :type arguments: dict
439
- :returns: Mapping of criterias with their corresponding values
440
- :rtype: dict
441
- """
442
- criterias: Dict[str, Any] = {}
443
- metadata_query_paths = get_metadata_query_paths(metadata_mapping)
444
- arguments_query_paths = get_arguments_query_paths(arguments)
445
- for query_path in arguments_query_paths:
446
- if query_path in metadata_query_paths:
447
- criteria_name = metadata_query_paths[query_path]
448
- else:
449
- # The criteria is custom and we must read
450
- # its name from the query path
451
- criteria_name = query_path.split(".")[1]
452
- criteria_value = arguments_query_paths[query_path]
453
- criterias[criteria_name] = criteria_value
454
- return criterias
455
-
456
-
457
- def search_products(
458
- product_type: str, arguments: Dict[str, Any], stac_formatted: bool = True
459
- ) -> Union[Dict[str, Any], SearchResult]:
460
- """Returns product search results
461
-
462
- :param product_type: The product type criteria
463
- :type product_type: str
464
- :param arguments: Request args
465
- :type arguments: dict
466
- :param stac_formatted: Whether input is STAC-formatted or not
467
- :type stac_formatted: bool
468
- :returns: A search result
469
- :rtype serialized GeoJSON response"""
470
-
471
- try:
472
- arg_product_type = arguments.pop("product_type", None)
473
- provider = arguments.pop("provider", None)
474
-
475
- unserialized = arguments.pop("unserialized", None)
476
-
477
- page, items_per_page = get_pagination_info(arguments)
478
- dtstart, dtend = get_datetime(arguments)
479
- geom = get_geometry(arguments)
480
-
481
- criterias = {
482
- "productType": product_type if product_type else arg_product_type,
483
- "page": page,
484
- "items_per_page": items_per_page,
485
- "start": dtstart,
486
- "end": dtend,
487
- "geom": geom,
488
- "provider": provider,
489
- }
490
-
491
- if stac_formatted:
492
- stac_provider_metadata_mapping = stac_provider_config.get("search", {}).get(
493
- "metadata_mapping", {}
494
- )
495
- extra_criterias = get_criterias_from_metadata_mapping(
496
- stac_provider_metadata_mapping, arguments
497
- )
498
- criterias.update(extra_criterias)
499
- else:
500
- criterias.update(arguments)
501
-
502
- if provider:
503
- criterias["raise_errors"] = True
504
-
505
- # We remove potential None values to use the default values of the search method
506
- criterias = dict((k, v) for k, v in criterias.items() if v is not None)
507
-
508
- products, total = eodag_api.search(**criterias)
509
-
510
- if not products and eodag_api.search_errors:
511
- search_error = RequestError(
512
- "No result could be obtained from any available provider and following "
513
- "error(s) appeared while searching:"
514
- )
515
- search_error.history = eodag_api.search_errors
516
- raise search_error
517
-
518
- products = filter_products(products, arguments, **criterias)
519
-
520
- response: Union[Dict[str, Any], SearchResult]
521
- if not unserialized:
522
- response = products.as_geojson_object()
523
- response.update(
524
- {
525
- "properties": {
526
- "page": page,
527
- "itemsPerPage": items_per_page,
528
- "totalResults": total,
529
- }
530
- }
531
- )
532
- else:
533
- response = products
534
- response.properties = {
535
- "page": page,
536
- "itemsPerPage": items_per_page,
537
- "totalResults": total,
538
- }
539
-
540
- except ValidationError as e:
541
- raise e
542
- except RuntimeError as e:
543
- raise e
544
- except UnsupportedProductType as e:
545
- raise e
546
-
547
- return response
548
-
549
-
550
- def search_product_by_id(
551
- uid: str,
552
- product_type: Optional[str] = None,
553
- provider: Optional[str] = None,
554
- **kwargs: Any,
555
- ) -> SearchResult:
556
- """Search a product by its id
557
-
558
- :param uid: The uid of the EO product
559
- :type uid: str
560
- :param product_type: (optional) The product type
561
- :type product_type: str
562
- :param provider: (optional) The provider to be used
563
- :type provider: str
564
- :param kwargs: additional search parameters
565
- :type kwargs: Any
566
- :returns: A search result
567
- :rtype: :class:`~eodag.api.search_result.SearchResult`
568
- :raises: :class:`~eodag.utils.exceptions.ValidationError`
569
- :raises: RuntimeError
570
- """
571
- if provider:
572
- kwargs["raise_errors"] = True
573
- try:
574
- products, _ = eodag_api.search(
575
- id=uid, productType=product_type, provider=provider, **kwargs
576
- )
577
- return products
578
- except ValidationError:
579
- raise
580
- except RuntimeError:
581
- raise
582
-
583
-
584
- # STAC ------------------------------------------------------------------------
585
-
586
-
587
- def get_stac_conformance() -> Dict[str, str]:
588
- """Build STAC conformance
589
-
590
- :returns: conformance dictionnary
591
- :rtype: dict
592
- """
593
- return stac_config["conformance"]
594
-
595
-
596
- def get_stac_api_version() -> str:
597
- """Get STAC API version
598
-
599
- :returns: STAC API version
600
- :rtype: str
601
- """
602
- return stac_config["stac_api_version"]
603
-
604
-
605
- def get_stac_collections(
606
- url: str, root: str, arguments: Dict[str, Any], provider: Optional[str] = None
607
- ) -> Dict[str, Any]:
608
- """Build STAC collections
609
-
610
- :param url: Requested URL
611
- :type url: str
612
- :param root: The API root
613
- :type root: str
614
- :param arguments: Request args
615
- :type arguments: dict
616
- :param provider: (optional) Chosen provider
617
- :type provider: str
618
- :returns: Collections dictionnary
619
- :rtype: dict
620
- """
621
- return StacCollection(
622
- url=url,
623
- stac_config=stac_config,
624
- provider=provider,
625
- eodag_api=eodag_api,
626
- root=root,
627
- ).get_collections(arguments)
628
-
629
-
630
- def get_stac_collection_by_id(
631
- url: str, root: str, collection_id: str, provider: Optional[str] = None
632
- ) -> Dict[str, Any]:
633
- """Build STAC collection by id
634
-
635
- :param url: Requested URL
636
- :type url: str
637
- :param root: API root
638
- :type root: str
639
- :param collection_id: Product_type as ID of the collection
640
- :type collection_id: str
641
- :param provider: (optional) Chosen provider
642
- :type provider: str
643
- :returns: Collection dictionary
644
- :rtype: dict
645
- """
646
- return StacCollection(
647
- url=url,
648
- stac_config=stac_config,
649
- provider=provider,
650
- eodag_api=eodag_api,
651
- root=root,
652
- ).get_collection_by_id(collection_id)
653
-
654
-
655
- def get_stac_item_by_id(
656
- url: str,
657
- item_id: str,
658
- catalogs: List[str],
659
- root: str = "/",
660
- provider: Optional[str] = None,
661
- **kwargs: Any,
662
- ) -> Dict[str, Any]:
663
- """Build STAC item by id
664
-
665
- :param url: Requested URL
666
- :type url: str
667
- :param item_id: Product ID
668
- :type item_id: str
669
- :param catalogs: Catalogs list (only first is used as product_type)
670
- :type catalogs: list
671
- :param root: (optional) API root
672
- :type root: str
673
- :param provider: (optional) Chosen provider
674
- :type provider: str
675
- :param kwargs: additional search parameters
676
- :type kwargs: Any
677
- :returns: Collection dictionary
678
- :rtype: dict
679
- """
680
- product_type = catalogs[0]
681
- _dc_qs = kwargs.get("_dc_qs", None)
682
-
683
- found_products = search_product_by_id(
684
- item_id, product_type=product_type, provider=provider, _dc_qs=_dc_qs
685
- )
686
-
687
- if len(found_products) > 0:
688
- found_products[0].product_type = eodag_api.get_alias_from_product_type(
689
- found_products[0].product_type
690
- )
691
- return StacItem(
692
- url=url,
693
- stac_config=stac_config,
694
- provider=provider,
695
- eodag_api=eodag_api,
696
- root=root,
697
- ).get_stac_item_from_product(product=found_products[0])
698
- else:
699
- return None
700
-
701
-
702
- def download_stac_item_by_id_stream(
703
- catalogs: List[str],
704
- item_id: str,
705
- provider: Optional[str] = None,
706
- asset: Optional[str] = None,
707
- **kwargs: Any,
708
- ) -> StreamingResponse:
709
- """Download item
710
-
711
- :param catalogs: Catalogs list (only first is used as product_type)
712
- :type catalogs: list
713
- :param item_id: Product ID
714
- :type item_id: str
715
- :param provider: (optional) Chosen provider
716
- :type provider: str
717
- :param kwargs: additional download parameters
718
- :type kwargs: Any
719
- :returns: a stream of the downloaded data (zip file)
720
- :rtype: StreamingResponse
721
- """
722
- product_type = catalogs[0]
723
- _dc_qs = kwargs.get("_dc_qs", None)
724
-
725
- search_plugin = next(
726
- eodag_api._plugins_manager.get_search_plugins(product_type, provider)
727
- )
728
- provider_product_type_config = search_plugin.config.products.get(
729
- product_type, {}
730
- ) or search_plugin.config.products.get(GENERIC_PRODUCT_TYPE, {})
731
- if provider_product_type_config.get("storeDownloadUrl", False):
732
- if item_id not in search_plugin.download_info:
733
- logger.error(f"data for item {item_id} not found")
734
- raise NotAvailableError(
735
- f"download url for product {item_id} could not be found, please redo "
736
- f"the search request to fetch the required data"
737
- )
738
- product_data = search_plugin.download_info[item_id]
739
- properties = {
740
- "id": item_id,
741
- "orderLink": product_data["orderLink"],
742
- "downloadLink": product_data["downloadLink"],
743
- "geometry": "-180 -90 180 90",
744
- }
745
- product = EOProduct(provider or product_data["provider"], properties)
746
- else:
747
-
748
- search_results = search_product_by_id(
749
- item_id, product_type=product_type, provider=provider, _dc_qs=_dc_qs
750
- )
751
- if len(search_results) > 0:
752
- product = search_results[0]
753
- else:
754
- raise NotAvailableError(
755
- f"Could not find {item_id} item in {product_type} collection for provider {provider}"
756
- )
757
-
758
- if product.downloader is None:
759
- download_plugin = eodag_api._plugins_manager.get_download_plugin(product)
760
- auth_plugin = eodag_api._plugins_manager.get_auth_plugin(
761
- download_plugin.provider
762
- )
763
- product.register_downloader(download_plugin, auth_plugin)
764
-
765
- auth = (
766
- product.downloader_auth.authenticate()
767
- if product.downloader_auth is not None
768
- else product.downloader_auth
769
- )
770
- try:
771
- download_stream_dict = product.downloader._stream_download_dict(
772
- product, auth=auth, asset=asset
773
- )
774
- except NotImplementedError:
775
- logger.warning(
776
- f"Download streaming not supported for {product.downloader}: downloading locally then delete"
777
- )
778
- product_path = eodag_api.download(product, extract=False, asset=asset)
779
- if os.path.isdir(product_path):
780
- # do not zip if dir contains only one file
781
- all_filenames = [
782
- f
783
- for f in glob.glob(
784
- os.path.join(product_path, "**", "*"), recursive=True
785
- )
786
- if os.path.isfile(f)
787
- ]
788
- if len(all_filenames) == 1:
789
- filepath_to_stream = all_filenames[0]
790
- else:
791
- filepath_to_stream = f"{product_path}.zip"
792
- logger.debug(
793
- f"Building archive for downloaded product path {filepath_to_stream}"
794
- )
795
- make_archive(product_path, "zip", product_path)
796
- rmtree(product_path)
797
- else:
798
- filepath_to_stream = product_path
799
-
800
- download_stream_dict = dict(
801
- content=read_file_chunks_and_delete(open(filepath_to_stream, "rb")),
802
- headers={
803
- "content-disposition": f"attachment; filename={os.path.basename(filepath_to_stream)}",
804
- },
805
- )
806
-
807
- return StreamingResponse(**download_stream_dict)
808
-
809
-
810
- def read_file_chunks_and_delete(
811
- opened_file: BufferedReader, chunk_size: int = 64 * 1024
812
- ) -> Iterator[bytes]:
813
- """Yield file chunks and delete file when finished."""
814
- while True:
815
- data = opened_file.read(chunk_size)
816
- if not data:
817
- opened_file.close()
818
- os.remove(opened_file.name)
819
- logger.debug(f"{opened_file.name} deleted after streaming complete")
820
- break
821
- yield data
822
- yield data
823
-
824
-
825
- def get_stac_catalogs(
826
- url: str,
827
- root: str = "/",
828
- catalogs: List[str] = [],
829
- provider: Optional[str] = None,
830
- fetch_providers: bool = True,
831
- ) -> Dict[str, Any]:
832
- """Build STAC catalog
833
-
834
- :param url: Requested URL
835
- :type url: str
836
- :param root: (optional) API root
837
- :type root: str
838
- :param catalogs: (optional) Catalogs list
839
- :type catalogs: list
840
- :param provider: (optional) Chosen provider
841
- :type provider: str
842
- :param fetch_providers: (optional) Whether to fetch providers for new product
843
- types or not
844
- :type fetch_providers: bool
845
- :returns: Catalog dictionary
846
- :rtype: dict
847
- """
848
- return StacCatalog(
849
- url=url,
850
- stac_config=stac_config,
851
- root=root,
852
- provider=provider,
853
- eodag_api=eodag_api,
854
- catalogs=catalogs,
855
- fetch_providers=fetch_providers,
856
- ).get_stac_catalog()
857
-
858
-
859
- def search_stac_items(
860
- url: str,
861
- arguments: Dict[str, Any],
862
- root: str = "/",
863
- catalogs: List[str] = [],
864
- provider: Optional[str] = None,
865
- method: Optional[str] = "GET",
866
- ) -> Dict[str, Any]:
867
- """Get items collection dict for given catalogs list
868
-
869
- :param url: Requested URL
870
- :type url: str
871
- :param arguments: Request args
872
- :type arguments: dict
873
- :param root: (optional) API root
874
- :type root: str
875
- :param catalogs: (optional) Catalogs list
876
- :type catalogs: list
877
- :param provider: (optional) Chosen provider
878
- :type provider: str
879
- :param method: (optional) search request HTTP method ('GET' or 'POST')
880
- :type method: str
881
- :returns: Catalog dictionnary
882
- :rtype: dict
883
- """
884
- collections = arguments.get("collections", None)
885
-
886
- catalog_url = url.replace("/items", "")
887
-
888
- next_page_kwargs = {
889
- key: value for key, value in arguments.copy().items() if value is not None
890
- }
891
- next_page_id = (
892
- int(next_page_kwargs["page"]) + 1 if "page" in next_page_kwargs else 2
893
- )
894
- next_page_kwargs["page"] = next_page_id
895
-
896
- # use catalogs from path or if it is empty, collections from args
897
- if catalogs:
898
- result_catalog = StacCatalog(
899
- url=catalog_url,
900
- stac_config=stac_config,
901
- root=root,
902
- provider=provider,
903
- eodag_api=eodag_api,
904
- catalogs=catalogs,
905
- )
906
- elif collections:
907
- # get collection as product_type
908
- if isinstance(collections, str):
909
- collections = collections.split(",")
910
- elif not isinstance(collections, list):
911
- raise ValidationError("Collections argument type should be Array")
912
-
913
- result_catalog = StacCatalog(
914
- stac_config=stac_config,
915
- root=root,
916
- provider=provider,
917
- eodag_api=eodag_api,
918
- # handle only one collection
919
- # per request (STAC allows multiple)
920
- catalogs=collections[0:1],
921
- url=catalog_url.replace("/search", f"/collections/{collections[0]}"),
922
- )
923
- arguments.pop("collections")
924
- else:
925
- raise NoMatchingProductType("Invalid request, collections argument is missing")
926
-
927
- # get products by ids
928
- ids = arguments.get("ids", None)
929
- if isinstance(ids, str):
930
- ids = [ids]
931
- if ids:
932
- search_results = SearchResult([])
933
- for item_id in ids:
934
- found_products = search_product_by_id(
935
- item_id, product_type=collections[0], provider=provider
936
- )
937
- if len(found_products) == 1:
938
- search_results.extend(found_products)
939
- search_results.properties = {
940
- "page": 1,
941
- "itemsPerPage": len(search_results),
942
- "totalResults": len(search_results),
943
- }
944
- else:
945
- if "datetime" in arguments.keys() and arguments["datetime"] is not None:
946
- arguments["dtstart"], arguments["dtend"] = get_datetime(arguments)
947
-
948
- search_products_arguments = dict(
949
- arguments,
950
- **result_catalog.search_args,
951
- **{"unserialized": "true", "provider": provider},
952
- )
953
-
954
- # check if time filtering appears both in search arguments and catalog
955
- # (for catalogs built by date: i.e. `year/2020/month/05`)
956
- if set(["dtstart", "dtend"]) <= set(arguments.keys()) and set(
957
- ["dtstart", "dtend"]
958
- ) <= set(result_catalog.search_args.keys()):
959
- search_date_min = (
960
- dateutil.parser.parse(arguments["dtstart"])
961
- if arguments["dtstart"]
962
- else datetime.datetime.min
963
- )
964
- search_date_max = (
965
- dateutil.parser.parse(arguments["dtend"])
966
- if arguments["dtend"]
967
- else datetime.datetime.now()
968
- )
969
- catalog_date_min = dateutil.parser.parse(
970
- result_catalog.search_args["dtstart"]
971
- )
972
- catalog_date_max = dateutil.parser.parse(
973
- result_catalog.search_args["dtend"]
974
- )
975
- # check if date intervals overlap
976
- if (search_date_min <= catalog_date_max) and (
977
- search_date_max >= catalog_date_min
978
- ):
979
- # use intersection
980
- search_products_arguments["dtstart"] = (
981
- max(search_date_min, catalog_date_min)
982
- .isoformat()
983
- .replace("+00:00", "")
984
- + "Z"
985
- )
986
- search_products_arguments["dtend"] = (
987
- min(search_date_max, catalog_date_max)
988
- .isoformat()
989
- .replace("+00:00", "")
990
- + "Z"
991
- )
992
- else:
993
- logger.warning("Time intervals do not overlap")
994
- # return empty results
995
- search_results = SearchResult([])
996
- search_results.properties = {
997
- "page": search_products_arguments.get("page", 1),
998
- "itemsPerPage": search_products_arguments.get(
999
- "itemsPerPage", DEFAULT_ITEMS_PER_PAGE
1000
- ),
1001
- "totalResults": 0,
1002
- }
1003
- return StacItem(
1004
- url=url,
1005
- stac_config=stac_config,
1006
- provider=provider,
1007
- eodag_api=eodag_api,
1008
- root=root,
1009
- ).get_stac_items(
1010
- search_results=search_results,
1011
- catalog=dict(
1012
- result_catalog.get_stac_catalog(),
1013
- **{"url": result_catalog.url, "root": result_catalog.root},
1014
- ),
1015
- )
1016
-
1017
- search_results = search_products(
1018
- product_type=result_catalog.search_args["product_type"],
1019
- arguments=search_products_arguments,
1020
- )
1021
-
1022
- for record in search_results:
1023
- record.product_type = eodag_api.get_alias_from_product_type(record.product_type)
1024
-
1025
- search_results.method = method
1026
- if method == "POST":
1027
- search_results.next = f"{url}"
1028
- search_results.body = next_page_kwargs
1029
-
1030
- elif method == "GET":
1031
- next_query_string = urlencode(next_page_kwargs)
1032
- search_results.next = f"{url}?{next_query_string}"
1033
-
1034
- items = StacItem(
1035
- url=url,
1036
- stac_config=stac_config,
1037
- provider=provider,
1038
- eodag_api=eodag_api,
1039
- root=root,
1040
- ).get_stac_items(
1041
- search_results=search_results,
1042
- catalog=dict(
1043
- result_catalog.get_stac_catalog(),
1044
- **{"url": result_catalog.url, "root": result_catalog.root},
1045
- ),
1046
- )
1047
-
1048
- return items
1049
-
1050
-
1051
- def get_stac_extension_oseo(url: str) -> Dict[str, str]:
1052
- """Build STAC OGC / OpenSearch Extension for EO
1053
-
1054
- :param url: Requested URL
1055
- :type url: str
1056
- :returns: Catalog dictionnary
1057
- :rtype: dict
1058
- """
1059
-
1060
- item_mapping = dict_items_recursive_apply(
1061
- stac_config["item"], lambda _, x: str(x).replace("$.product.", "$.")
1062
- )
1063
-
1064
- # all properties as string type by default
1065
- oseo_properties = {
1066
- "oseo:{}".format(k): {
1067
- "type": "string",
1068
- "title": k[0].upper() + re.sub(r"([A-Z][a-z]+)", r" \1", k[1:]),
1069
- }
1070
- for k, v in OSEO_METADATA_MAPPING.items()
1071
- if v not in str(item_mapping)
1072
- }
1073
-
1074
- return StacCommon.get_stac_extension(
1075
- url=url, stac_config=stac_config, extension="oseo", properties=oseo_properties
1076
- )
1077
-
1078
-
1079
- def fetch_collection_queryable_properties(
1080
- collection_id: Optional[str] = None, provider: Optional[str] = None, **kwargs: Any
1081
- ) -> Dict[str, StacQueryableProperty]:
1082
- """Fetch the queryable properties for a collection.
1083
-
1084
- :param collection_id: The ID of the collection.
1085
- :type collection_id: str
1086
- :param provider: (optional) The provider.
1087
- :type provider: str
1088
- :param kwargs: additional filters for queryables (`productType` or other search
1089
- arguments)
1090
- :type kwargs: Any
1091
- :returns: A set containing the STAC standardized queryable properties for a collection.
1092
- :rtype Dict[str, StacQueryableProperty]: set
1093
- """
1094
- if not collection_id and "collections" in kwargs:
1095
- collection_ids = kwargs.pop("collections").split(",")
1096
- collection_id = collection_ids[0]
1097
-
1098
- if collection_id and "productType" in kwargs:
1099
- kwargs.pop("productType")
1100
- elif "productType" in kwargs:
1101
- collection_id = kwargs.pop("productType")
1102
-
1103
- if "ids" in kwargs:
1104
- kwargs["id"] = kwargs.pop("ids")
1105
-
1106
- if "datetime" in kwargs:
1107
- dates = get_datetime(kwargs)
1108
- kwargs["start"] = dates[0]
1109
- kwargs["end"] = dates[1]
1110
-
1111
- python_queryables = eodag_api.list_queryables(
1112
- provider=provider, productType=collection_id, **kwargs
1113
- )
1114
- python_queryables.pop("start")
1115
- python_queryables.pop("end")
1116
-
1117
- stac_queryables = dict()
1118
- for param, queryable in python_queryables.items():
1119
- stac_param = EODAGSearch.to_stac(param)
1120
- stac_queryables[
1121
- stac_param
1122
- ] = StacQueryableProperty.from_python_field_definition(stac_param, queryable)
1123
-
1124
- return stac_queryables
1125
-
1126
-
1127
- def eodag_api_init() -> None:
1128
- """Init EODataAccessGateway server instance, pre-running all time consuming tasks"""
1129
- eodag_api.fetch_product_types_list()
1130
-
1131
- # pre-build search plugins
1132
- for provider in eodag_api.available_providers():
1133
- next(eodag_api._plugins_manager.get_search_plugins(provider=provider))