eodag 3.0.1__py3-none-any.whl → 3.1.0b1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- eodag/api/core.py +116 -86
- eodag/api/product/_assets.py +6 -6
- eodag/api/product/_product.py +18 -18
- eodag/api/product/metadata_mapping.py +39 -11
- eodag/cli.py +22 -1
- eodag/config.py +14 -14
- eodag/plugins/apis/ecmwf.py +37 -14
- eodag/plugins/apis/usgs.py +5 -5
- eodag/plugins/authentication/openid_connect.py +2 -2
- eodag/plugins/authentication/token.py +37 -6
- eodag/plugins/crunch/filter_property.py +2 -3
- eodag/plugins/download/aws.py +11 -12
- eodag/plugins/download/base.py +30 -39
- eodag/plugins/download/creodias_s3.py +29 -0
- eodag/plugins/download/http.py +144 -152
- eodag/plugins/download/s3rest.py +5 -7
- eodag/plugins/search/base.py +73 -25
- eodag/plugins/search/build_search_result.py +1047 -310
- eodag/plugins/search/creodias_s3.py +25 -19
- eodag/plugins/search/data_request_search.py +1 -1
- eodag/plugins/search/qssearch.py +51 -139
- eodag/resources/ext_product_types.json +1 -1
- eodag/resources/product_types.yml +391 -32
- eodag/resources/providers.yml +678 -1744
- eodag/rest/core.py +92 -62
- eodag/rest/server.py +31 -4
- eodag/rest/types/eodag_search.py +6 -0
- eodag/rest/types/queryables.py +5 -6
- eodag/rest/utils/__init__.py +3 -0
- eodag/types/__init__.py +56 -15
- eodag/types/download_args.py +2 -2
- eodag/types/queryables.py +180 -72
- eodag/types/whoosh.py +126 -0
- eodag/utils/__init__.py +71 -10
- eodag/utils/exceptions.py +27 -20
- eodag/utils/repr.py +65 -6
- eodag/utils/requests.py +11 -11
- {eodag-3.0.1.dist-info → eodag-3.1.0b1.dist-info}/METADATA +76 -76
- {eodag-3.0.1.dist-info → eodag-3.1.0b1.dist-info}/RECORD +43 -44
- {eodag-3.0.1.dist-info → eodag-3.1.0b1.dist-info}/WHEEL +1 -1
- {eodag-3.0.1.dist-info → eodag-3.1.0b1.dist-info}/entry_points.txt +3 -2
- eodag/utils/constraints.py +0 -244
- {eodag-3.0.1.dist-info → eodag-3.1.0b1.dist-info}/LICENSE +0 -0
- {eodag-3.0.1.dist-info → eodag-3.1.0b1.dist-info}/top_level.txt +0 -0
eodag/api/core.py
CHANGED
|
@@ -24,27 +24,15 @@ import re
|
|
|
24
24
|
import shutil
|
|
25
25
|
import tempfile
|
|
26
26
|
from operator import itemgetter
|
|
27
|
-
from typing import
|
|
28
|
-
TYPE_CHECKING,
|
|
29
|
-
Annotated,
|
|
30
|
-
Any,
|
|
31
|
-
Dict,
|
|
32
|
-
Iterator,
|
|
33
|
-
List,
|
|
34
|
-
Optional,
|
|
35
|
-
Set,
|
|
36
|
-
Tuple,
|
|
37
|
-
Union,
|
|
38
|
-
)
|
|
27
|
+
from typing import TYPE_CHECKING, Any, Dict, Iterator, List, Optional, Tuple, Union
|
|
39
28
|
|
|
40
29
|
import geojson
|
|
41
30
|
import pkg_resources
|
|
42
31
|
import yaml.parser
|
|
43
32
|
from pkg_resources import resource_filename
|
|
44
|
-
from pydantic.fields import FieldInfo
|
|
45
33
|
from whoosh import analysis, fields
|
|
46
34
|
from whoosh.fields import Schema
|
|
47
|
-
from whoosh.index import
|
|
35
|
+
from whoosh.index import exists_in, open_dir
|
|
48
36
|
from whoosh.qparser import QueryParser
|
|
49
37
|
|
|
50
38
|
from eodag.api.product.metadata_mapping import (
|
|
@@ -69,11 +57,11 @@ from eodag.config import (
|
|
|
69
57
|
)
|
|
70
58
|
from eodag.plugins.manager import PluginManager
|
|
71
59
|
from eodag.plugins.search import PreparedSearch
|
|
72
|
-
from eodag.plugins.search.build_search_result import
|
|
60
|
+
from eodag.plugins.search.build_search_result import MeteoblueSearch
|
|
73
61
|
from eodag.plugins.search.qssearch import PostJsonSearch
|
|
74
62
|
from eodag.types import model_fields_to_annotated
|
|
75
|
-
from eodag.types.queryables import CommonQueryables
|
|
76
|
-
from eodag.types.whoosh import EODAGQueryParser
|
|
63
|
+
from eodag.types.queryables import CommonQueryables, QueryablesDict
|
|
64
|
+
from eodag.types.whoosh import EODAGQueryParser, create_in
|
|
77
65
|
from eodag.utils import (
|
|
78
66
|
DEFAULT_DOWNLOAD_TIMEOUT,
|
|
79
67
|
DEFAULT_DOWNLOAD_WAIT,
|
|
@@ -84,7 +72,6 @@ from eodag.utils import (
|
|
|
84
72
|
HTTP_REQ_TIMEOUT,
|
|
85
73
|
MockResponse,
|
|
86
74
|
_deprecated,
|
|
87
|
-
copy_deepcopy,
|
|
88
75
|
get_geometry_from_various,
|
|
89
76
|
makedirs,
|
|
90
77
|
obj_md5sum,
|
|
@@ -93,6 +80,7 @@ from eodag.utils import (
|
|
|
93
80
|
uri_to_path,
|
|
94
81
|
)
|
|
95
82
|
from eodag.utils.exceptions import (
|
|
83
|
+
AuthenticationError,
|
|
96
84
|
EodagError,
|
|
97
85
|
NoMatchingProductType,
|
|
98
86
|
PluginImplementationError,
|
|
@@ -219,9 +207,10 @@ class EODataAccessGateway:
|
|
|
219
207
|
"eodag",
|
|
220
208
|
os.path.join("resources", "locations_conf_template.yml"),
|
|
221
209
|
)
|
|
222
|
-
with
|
|
223
|
-
|
|
224
|
-
|
|
210
|
+
with (
|
|
211
|
+
open(locations_conf_template) as infile,
|
|
212
|
+
open(locations_conf_path, "w") as outfile,
|
|
213
|
+
):
|
|
225
214
|
# The template contains paths in the form of:
|
|
226
215
|
# /path/to/locations/file.shp
|
|
227
216
|
path_template = "/path/to/locations/"
|
|
@@ -317,13 +306,18 @@ class EODataAccessGateway:
|
|
|
317
306
|
product_type, **{"md5": self.product_types_config_md5}
|
|
318
307
|
)
|
|
319
308
|
# add to index
|
|
320
|
-
|
|
321
|
-
|
|
322
|
-
|
|
323
|
-
|
|
324
|
-
|
|
325
|
-
|
|
326
|
-
|
|
309
|
+
try:
|
|
310
|
+
ix_writer.add_document(
|
|
311
|
+
**{
|
|
312
|
+
k: v
|
|
313
|
+
for k, v in versioned_product_type.items()
|
|
314
|
+
if k in product_types_schema.names()
|
|
315
|
+
}
|
|
316
|
+
)
|
|
317
|
+
except TypeError as e:
|
|
318
|
+
logger.error(
|
|
319
|
+
f"Cannot write product type {product_type['ID']} into index. e={e} product_type={product_type}"
|
|
320
|
+
)
|
|
327
321
|
ix_writer.commit()
|
|
328
322
|
|
|
329
323
|
def set_preferred_provider(self, provider: str) -> None:
|
|
@@ -1773,12 +1767,12 @@ class EODataAccessGateway:
|
|
|
1773
1767
|
for plugin in self._plugins_manager.get_search_plugins(
|
|
1774
1768
|
product_type=product_type, provider=provider
|
|
1775
1769
|
):
|
|
1776
|
-
# exclude
|
|
1770
|
+
# exclude MeteoblueSearch plugins from search fallback for unknown product_type
|
|
1777
1771
|
if (
|
|
1778
1772
|
provider != plugin.provider
|
|
1779
1773
|
and preferred_provider != plugin.provider
|
|
1780
1774
|
and product_type not in self.product_types_config
|
|
1781
|
-
and isinstance(plugin,
|
|
1775
|
+
and isinstance(plugin, MeteoblueSearch)
|
|
1782
1776
|
):
|
|
1783
1777
|
continue
|
|
1784
1778
|
search_plugins.append(plugin)
|
|
@@ -1801,27 +1795,7 @@ class EODataAccessGateway:
|
|
|
1801
1795
|
# Add product_types_config to plugin config. This dict contains product
|
|
1802
1796
|
# type metadata that will also be stored in each product's properties.
|
|
1803
1797
|
for search_plugin in search_plugins:
|
|
1804
|
-
|
|
1805
|
-
search_plugin.config.product_type_config = dict(
|
|
1806
|
-
[
|
|
1807
|
-
p
|
|
1808
|
-
for p in self.list_product_types(
|
|
1809
|
-
search_plugin.provider, fetch_providers=False
|
|
1810
|
-
)
|
|
1811
|
-
if p["_id"] == product_type
|
|
1812
|
-
][0],
|
|
1813
|
-
**{"productType": product_type},
|
|
1814
|
-
)
|
|
1815
|
-
# If the product isn't in the catalog, it's a generic product type.
|
|
1816
|
-
except IndexError:
|
|
1817
|
-
# Construct the GENERIC_PRODUCT_TYPE metadata
|
|
1818
|
-
search_plugin.config.product_type_config = dict(
|
|
1819
|
-
ID=GENERIC_PRODUCT_TYPE,
|
|
1820
|
-
**self.product_types_config[GENERIC_PRODUCT_TYPE],
|
|
1821
|
-
productType=product_type,
|
|
1822
|
-
)
|
|
1823
|
-
# Remove the ID since this is equal to productType.
|
|
1824
|
-
search_plugin.config.product_type_config.pop("ID", None)
|
|
1798
|
+
self._attach_product_type_config(search_plugin, product_type)
|
|
1825
1799
|
|
|
1826
1800
|
return search_plugins, kwargs
|
|
1827
1801
|
|
|
@@ -2038,8 +2012,8 @@ class EODataAccessGateway:
|
|
|
2038
2012
|
search_result: SearchResult,
|
|
2039
2013
|
downloaded_callback: Optional[DownloadedCallback] = None,
|
|
2040
2014
|
progress_callback: Optional[ProgressCallback] = None,
|
|
2041
|
-
wait:
|
|
2042
|
-
timeout:
|
|
2015
|
+
wait: float = DEFAULT_DOWNLOAD_WAIT,
|
|
2016
|
+
timeout: float = DEFAULT_DOWNLOAD_TIMEOUT,
|
|
2043
2017
|
**kwargs: Unpack[DownloadConf],
|
|
2044
2018
|
) -> List[str]:
|
|
2045
2019
|
"""Download all products resulting from a search.
|
|
@@ -2201,8 +2175,8 @@ class EODataAccessGateway:
|
|
|
2201
2175
|
self,
|
|
2202
2176
|
product: EOProduct,
|
|
2203
2177
|
progress_callback: Optional[ProgressCallback] = None,
|
|
2204
|
-
wait:
|
|
2205
|
-
timeout:
|
|
2178
|
+
wait: float = DEFAULT_DOWNLOAD_WAIT,
|
|
2179
|
+
timeout: float = DEFAULT_DOWNLOAD_TIMEOUT,
|
|
2206
2180
|
**kwargs: Unpack[DownloadConf],
|
|
2207
2181
|
) -> str:
|
|
2208
2182
|
"""Download a single product.
|
|
@@ -2280,67 +2254,96 @@ class EODataAccessGateway:
|
|
|
2280
2254
|
return self._plugins_manager.get_crunch_plugin(name, **plugin_conf)
|
|
2281
2255
|
|
|
2282
2256
|
def list_queryables(
|
|
2283
|
-
self,
|
|
2284
|
-
|
|
2257
|
+
self,
|
|
2258
|
+
provider: Optional[str] = None,
|
|
2259
|
+
fetch_providers: bool = True,
|
|
2260
|
+
**kwargs: Any,
|
|
2261
|
+
) -> QueryablesDict:
|
|
2285
2262
|
"""Fetch the queryable properties for a given product type and/or provider.
|
|
2286
2263
|
|
|
2287
2264
|
:param provider: (optional) The provider.
|
|
2265
|
+
:param fetch_providers: If new product types should be fetched from the providers; default: True
|
|
2288
2266
|
:param kwargs: additional filters for queryables (`productType` or other search
|
|
2289
2267
|
arguments)
|
|
2290
2268
|
|
|
2291
2269
|
:raises UnsupportedProductType: If the specified product type is not available for the
|
|
2292
2270
|
provider.
|
|
2293
2271
|
|
|
2294
|
-
:returns: A
|
|
2295
|
-
parameters to their annotated type
|
|
2272
|
+
:returns: A :class:`~eodag.api.product.queryables.QuerybalesDict` containing the EODAG queryable
|
|
2273
|
+
properties, associating parameters to their annotated type, and a additional_properties attribute
|
|
2296
2274
|
"""
|
|
2275
|
+
# only fetch providers if product type is not found
|
|
2297
2276
|
available_product_types = [
|
|
2298
2277
|
pt["ID"]
|
|
2299
2278
|
for pt in self.list_product_types(provider=provider, fetch_providers=False)
|
|
2300
2279
|
]
|
|
2301
|
-
product_type = kwargs.get("productType")
|
|
2280
|
+
product_type: Optional[str] = kwargs.get("productType")
|
|
2281
|
+
pt_alias: Optional[str] = product_type
|
|
2302
2282
|
|
|
2303
2283
|
if product_type:
|
|
2284
|
+
if product_type not in available_product_types:
|
|
2285
|
+
if fetch_providers:
|
|
2286
|
+
# fetch providers and try again
|
|
2287
|
+
available_product_types = [
|
|
2288
|
+
pt["ID"]
|
|
2289
|
+
for pt in self.list_product_types(
|
|
2290
|
+
provider=provider, fetch_providers=True
|
|
2291
|
+
)
|
|
2292
|
+
]
|
|
2293
|
+
raise UnsupportedProductType(f"{product_type} is not available.")
|
|
2304
2294
|
try:
|
|
2305
2295
|
kwargs["productType"] = product_type = self.get_product_type_from_alias(
|
|
2306
2296
|
product_type
|
|
2307
2297
|
)
|
|
2308
2298
|
except NoMatchingProductType as e:
|
|
2309
|
-
raise UnsupportedProductType(f"{product_type} is not available") from e
|
|
2310
|
-
|
|
2311
|
-
if product_type and product_type not in available_product_types:
|
|
2312
|
-
self.fetch_product_types_list()
|
|
2299
|
+
raise UnsupportedProductType(f"{product_type} is not available.") from e
|
|
2313
2300
|
|
|
2314
2301
|
if not provider and not product_type:
|
|
2315
|
-
return
|
|
2302
|
+
return QueryablesDict(
|
|
2303
|
+
additional_properties=True,
|
|
2304
|
+
**model_fields_to_annotated(CommonQueryables.model_fields),
|
|
2305
|
+
)
|
|
2316
2306
|
|
|
2317
|
-
|
|
2307
|
+
queryables: QueryablesDict = QueryablesDict(
|
|
2308
|
+
additional_properties=True, additional_information="", **{}
|
|
2309
|
+
)
|
|
2318
2310
|
|
|
2319
2311
|
for plugin in self._plugins_manager.get_search_plugins(product_type, provider):
|
|
2312
|
+
# attach product type config
|
|
2313
|
+
product_type_configs = {}
|
|
2314
|
+
if product_type:
|
|
2315
|
+
self._attach_product_type_config(plugin, product_type)
|
|
2316
|
+
product_type_configs[product_type] = plugin.config.product_type_config
|
|
2317
|
+
else:
|
|
2318
|
+
for pt in available_product_types:
|
|
2319
|
+
self._attach_product_type_config(plugin, pt)
|
|
2320
|
+
product_type_configs[pt] = plugin.config.product_type_config
|
|
2321
|
+
|
|
2322
|
+
# authenticate if required
|
|
2320
2323
|
if getattr(plugin.config, "need_auth", False) and (
|
|
2321
2324
|
auth := self._plugins_manager.get_auth_plugin(plugin)
|
|
2322
2325
|
):
|
|
2323
|
-
|
|
2324
|
-
|
|
2325
|
-
|
|
2326
|
+
try:
|
|
2327
|
+
plugin.auth = auth.authenticate()
|
|
2328
|
+
except AuthenticationError:
|
|
2329
|
+
logger.debug(
|
|
2330
|
+
"queryables from provider %s could not be fetched due to an authentication error",
|
|
2331
|
+
plugin.provider,
|
|
2332
|
+
)
|
|
2333
|
+
plugin_queryables = plugin.list_queryables(
|
|
2334
|
+
kwargs,
|
|
2335
|
+
available_product_types,
|
|
2336
|
+
product_type_configs,
|
|
2337
|
+
product_type,
|
|
2338
|
+
pt_alias,
|
|
2326
2339
|
)
|
|
2327
|
-
|
|
2328
|
-
|
|
2329
|
-
|
|
2330
|
-
|
|
2331
|
-
|
|
2332
|
-
|
|
2333
|
-
|
|
2334
|
-
if k in queryable_keys
|
|
2335
|
-
}
|
|
2336
|
-
|
|
2337
|
-
# always keep at least CommonQueryables
|
|
2338
|
-
common_queryables = copy_deepcopy(CommonQueryables.model_fields)
|
|
2339
|
-
for key, queryable in common_queryables.items():
|
|
2340
|
-
if key in kwargs:
|
|
2341
|
-
queryable.default = kwargs[key]
|
|
2342
|
-
|
|
2343
|
-
queryables.update(model_fields_to_annotated(common_queryables))
|
|
2340
|
+
queryables.update(plugin_queryables)
|
|
2341
|
+
if not plugin_queryables.additional_properties:
|
|
2342
|
+
queryables.additional_properties = False
|
|
2343
|
+
if plugin_queryables.additional_information:
|
|
2344
|
+
queryables.additional_information += (
|
|
2345
|
+
f"{provider}: {plugin_queryables.additional_information}"
|
|
2346
|
+
)
|
|
2344
2347
|
|
|
2345
2348
|
return queryables
|
|
2346
2349
|
|
|
@@ -2375,3 +2378,30 @@ class EODataAccessGateway:
|
|
|
2375
2378
|
],
|
|
2376
2379
|
}
|
|
2377
2380
|
return sortables
|
|
2381
|
+
|
|
2382
|
+
def _attach_product_type_config(self, plugin: Search, product_type: str) -> None:
|
|
2383
|
+
"""
|
|
2384
|
+
Attach product_types_config to plugin config. This dict contains product
|
|
2385
|
+
type metadata that will also be stored in each product's properties.
|
|
2386
|
+
"""
|
|
2387
|
+
try:
|
|
2388
|
+
plugin.config.product_type_config = dict(
|
|
2389
|
+
[
|
|
2390
|
+
p
|
|
2391
|
+
for p in self.list_product_types(
|
|
2392
|
+
plugin.provider, fetch_providers=False
|
|
2393
|
+
)
|
|
2394
|
+
if p["_id"] == product_type
|
|
2395
|
+
][0],
|
|
2396
|
+
**{"productType": product_type},
|
|
2397
|
+
)
|
|
2398
|
+
# If the product isn't in the catalog, it's a generic product type.
|
|
2399
|
+
except IndexError:
|
|
2400
|
+
# Construct the GENERIC_PRODUCT_TYPE metadata
|
|
2401
|
+
plugin.config.product_type_config = dict(
|
|
2402
|
+
ID=GENERIC_PRODUCT_TYPE,
|
|
2403
|
+
**self.product_types_config[GENERIC_PRODUCT_TYPE],
|
|
2404
|
+
productType=product_type,
|
|
2405
|
+
)
|
|
2406
|
+
# Remove the ID since this is equal to productType.
|
|
2407
|
+
plugin.config.product_type_config.pop("ID", None)
|
eodag/api/product/_assets.py
CHANGED
|
@@ -98,12 +98,12 @@ class AssetsDict(UserDict):
|
|
|
98
98
|
<details><summary style='color: grey;'>
|
|
99
99
|
<span style='color: black'>'{k}'</span>: 
|
|
100
100
|
{{
|
|
101
|
-
{"'roles': '<span style='color: black'>"+str(v['roles'])+"</span>', "
|
|
102
|
-
|
|
103
|
-
{"'type': '"+str(v['type'])+"', "
|
|
104
|
-
|
|
105
|
-
{"'title': '<span style='color: black'>"+str(v['title'])+"</span>', "
|
|
106
|
-
|
|
101
|
+
{"'roles': '<span style='color: black'>" + str(v['roles']) + "</span>', "
|
|
102
|
+
if v.get("roles") else ""}
|
|
103
|
+
{"'type': '" + str(v['type']) + "', "
|
|
104
|
+
if v.get("type") else ""}
|
|
105
|
+
{"'title': '<span style='color: black'>" + str(v['title']) + "</span>', "
|
|
106
|
+
if v.get("title") else ""}
|
|
107
107
|
...
|
|
108
108
|
}}
|
|
109
109
|
</summary>
|
eodag/api/product/_product.py
CHANGED
|
@@ -116,6 +116,7 @@ class EOProduct:
|
|
|
116
116
|
properties: Dict[str, Any]
|
|
117
117
|
product_type: Optional[str]
|
|
118
118
|
location: str
|
|
119
|
+
filename: str
|
|
119
120
|
remote_location: str
|
|
120
121
|
search_kwargs: Any
|
|
121
122
|
geometry: BaseGeometry
|
|
@@ -281,8 +282,8 @@ class EOProduct:
|
|
|
281
282
|
def download(
|
|
282
283
|
self,
|
|
283
284
|
progress_callback: Optional[ProgressCallback] = None,
|
|
284
|
-
wait:
|
|
285
|
-
timeout:
|
|
285
|
+
wait: float = DEFAULT_DOWNLOAD_WAIT,
|
|
286
|
+
timeout: float = DEFAULT_DOWNLOAD_TIMEOUT,
|
|
286
287
|
**kwargs: Unpack[DownloadConf],
|
|
287
288
|
) -> str:
|
|
288
289
|
"""Download the EO product using the provided download plugin and the
|
|
@@ -525,22 +526,21 @@ class EOProduct:
|
|
|
525
526
|
<tr style='background-color: transparent;'>
|
|
526
527
|
<td style='text-align: left; vertical-align: top;'>
|
|
527
528
|
{dict_to_html_table({
|
|
528
|
-
|
|
529
|
-
|
|
530
|
-
|
|
531
|
-
|
|
532
|
-
|
|
533
|
-
|
|
534
|
-
|
|
535
|
-
|
|
536
|
-
|
|
537
|
-
|
|
538
|
-
<details><summary style='color: grey; margin-top: 10px;'>properties: ({
|
|
539
|
-
|
|
540
|
-
|
|
541
|
-
<details><summary style='color: grey; margin-top: 10px;'>assets: ({
|
|
542
|
-
|
|
543
|
-
})</summary>{self.assets._repr_html_(embeded=True)}</details>
|
|
529
|
+
"provider": self.provider,
|
|
530
|
+
"product_type": self.product_type,
|
|
531
|
+
"properties["id"]": self.properties.get('id', None),
|
|
532
|
+
"properties["startTimeFromAscendingNode"]": self.properties.get(
|
|
533
|
+
'startTimeFromAscendingNode', None
|
|
534
|
+
),
|
|
535
|
+
"properties["completionTimeFromAscendingNode"]": self.properties.get(
|
|
536
|
+
'completionTimeFromAscendingNode', None
|
|
537
|
+
),
|
|
538
|
+
}, brackets=False)}
|
|
539
|
+
<details><summary style='color: grey; margin-top: 10px;'>properties: ({len(
|
|
540
|
+
self.properties)})</summary>{
|
|
541
|
+
dict_to_html_table(self.properties, depth=1)}</details>
|
|
542
|
+
<details><summary style='color: grey; margin-top: 10px;'>assets: ({len(
|
|
543
|
+
self.assets)})</summary>{self.assets._repr_html_(embeded=True)}</details>
|
|
544
544
|
</td>
|
|
545
545
|
<td {geom_style} title='geometry'>geometry<br />{self.geometry._repr_svg_()}</td>
|
|
546
546
|
<td {thumbnail_style} title='properties["thumbnail"]'>{thumbnail_html}</td>
|
|
@@ -40,6 +40,7 @@ from typing import (
|
|
|
40
40
|
import geojson
|
|
41
41
|
import orjson
|
|
42
42
|
import pyproj
|
|
43
|
+
import shapely
|
|
43
44
|
from dateutil.parser import isoparse
|
|
44
45
|
from dateutil.relativedelta import relativedelta
|
|
45
46
|
from dateutil.tz import UTC, tzutc
|
|
@@ -179,6 +180,8 @@ def format_metadata(search_param: str, *args: Any, **kwargs: Any) -> str:
|
|
|
179
180
|
- ``recursive_sub_str``: recursively substitue in the structure (e.g. dict)
|
|
180
181
|
values matching a regex
|
|
181
182
|
- ``slice_str``: slice a string (equivalent to s[start, end, step])
|
|
183
|
+
- ``to_lower``: Convert a string to lowercase
|
|
184
|
+
- ``to_upper``: Convert a string to uppercase
|
|
182
185
|
- ``fake_l2a_title_from_l1c``: used to generate SAFE format metadata for data from AWS
|
|
183
186
|
- ``s2msil2a_title_to_aws_productinfo``: used to generate SAFE format metadata for data from AWS
|
|
184
187
|
- ``split_cop_dem_id``: get the bbox by splitting the product id
|
|
@@ -363,6 +366,8 @@ def format_metadata(search_param: str, *args: Any, **kwargs: Any) -> str:
|
|
|
363
366
|
|
|
364
367
|
@staticmethod
|
|
365
368
|
def convert_to_nwse_bounds(input_geom: BaseGeometry) -> List[float]:
|
|
369
|
+
if isinstance(input_geom, str):
|
|
370
|
+
input_geom = shapely.wkt.loads(input_geom)
|
|
366
371
|
return list(input_geom.bounds[-1:] + input_geom.bounds[:-1])
|
|
367
372
|
|
|
368
373
|
@staticmethod
|
|
@@ -558,6 +563,16 @@ def format_metadata(search_param: str, *args: Any, **kwargs: Any) -> str:
|
|
|
558
563
|
]
|
|
559
564
|
return string[cmin:cmax:cstep]
|
|
560
565
|
|
|
566
|
+
@staticmethod
|
|
567
|
+
def convert_to_lower(string: str) -> str:
|
|
568
|
+
"""Convert a string to lowercase."""
|
|
569
|
+
return string.lower()
|
|
570
|
+
|
|
571
|
+
@staticmethod
|
|
572
|
+
def convert_to_upper(string: str) -> str:
|
|
573
|
+
"""Convert a string to uppercase."""
|
|
574
|
+
return string.upper()
|
|
575
|
+
|
|
561
576
|
@staticmethod
|
|
562
577
|
def convert_fake_l2a_title_from_l1c(string: str) -> str:
|
|
563
578
|
id_regex = re.compile(
|
|
@@ -834,7 +849,7 @@ def format_metadata(search_param: str, *args: Any, **kwargs: Any) -> str:
|
|
|
834
849
|
date_object = datetime.strptime(utc_date, "%Y-%m-%dT%H:%M:%S.%fZ")
|
|
835
850
|
date_object_second_year = date_object + relativedelta(years=1)
|
|
836
851
|
return [
|
|
837
|
-
f
|
|
852
|
+
f"{date_object.strftime('%Y')}_{date_object_second_year.strftime('%y')}"
|
|
838
853
|
]
|
|
839
854
|
|
|
840
855
|
@staticmethod
|
|
@@ -902,10 +917,8 @@ def format_metadata(search_param: str, *args: Any, **kwargs: Any) -> str:
|
|
|
902
917
|
return assets_dict
|
|
903
918
|
|
|
904
919
|
# if stac extension colon separator `:` is in search params, parse it to prevent issues with vformat
|
|
905
|
-
if re.search(r"{[
|
|
906
|
-
search_param = re.sub(
|
|
907
|
-
r"{([a-zA-Z0-9_-]*):([a-zA-Z0-9_-]*)}", r"{\1_COLON_\2}", search_param
|
|
908
|
-
)
|
|
920
|
+
if re.search(r"{[\w-]*:[\w#-]*}", search_param):
|
|
921
|
+
search_param = re.sub(r"{([\w-]*):([\w#-]*)}", r"{\1_COLON_\2}", search_param)
|
|
909
922
|
kwargs = {k.replace(":", "_COLON_"): v for k, v in kwargs.items()}
|
|
910
923
|
|
|
911
924
|
return MetadataFormatter().vformat(search_param, args, kwargs)
|
|
@@ -975,10 +988,24 @@ def properties_from_json(
|
|
|
975
988
|
if re.search(r"({[^{}:]+})+", conversion_or_none):
|
|
976
989
|
conversion_or_none = conversion_or_none.format(**properties)
|
|
977
990
|
|
|
978
|
-
|
|
979
|
-
|
|
980
|
-
|
|
981
|
-
|
|
991
|
+
if extracted_value == NOT_AVAILABLE:
|
|
992
|
+
# try if value can be formatted even if it is not available
|
|
993
|
+
try:
|
|
994
|
+
properties[metadata] = format_metadata(
|
|
995
|
+
"{%s%s%s}" % (metadata, SEP, conversion_or_none),
|
|
996
|
+
**{metadata: extracted_value},
|
|
997
|
+
)
|
|
998
|
+
except ValueError:
|
|
999
|
+
logger.debug(
|
|
1000
|
+
f"{metadata}: {extracted_value} could not be formatted with {conversion_or_none}"
|
|
1001
|
+
)
|
|
1002
|
+
continue
|
|
1003
|
+
else:
|
|
1004
|
+
# in this case formatting should work, otherwise something is wrong in the mapping
|
|
1005
|
+
properties[metadata] = format_metadata(
|
|
1006
|
+
"{%s%s%s}" % (metadata, SEP, conversion_or_none),
|
|
1007
|
+
**{metadata: extracted_value},
|
|
1008
|
+
)
|
|
982
1009
|
# properties as python objects when possible (format_metadata returns only strings)
|
|
983
1010
|
try:
|
|
984
1011
|
properties[metadata] = ast.literal_eval(properties[metadata])
|
|
@@ -1462,7 +1489,7 @@ def get_queryable_from_provider(
|
|
|
1462
1489
|
:param metadata_mapping: metadata-mapping configuration
|
|
1463
1490
|
:returns: EODAG configured queryable parameter or None
|
|
1464
1491
|
"""
|
|
1465
|
-
pattern = rf"\
|
|
1492
|
+
pattern = rf"\"{provider_queryable}\""
|
|
1466
1493
|
# if 1:1 mapping exists privilege this one instead of other mapping
|
|
1467
1494
|
# e.g. provider queryable = year -> use year and not date in which year also appears
|
|
1468
1495
|
mapping_values = [
|
|
@@ -1498,7 +1525,8 @@ def get_provider_queryable_key(
|
|
|
1498
1525
|
provider_queryables: Dict[str, Any],
|
|
1499
1526
|
metadata_mapping: Dict[str, Union[List[Any], str]],
|
|
1500
1527
|
) -> str:
|
|
1501
|
-
"""
|
|
1528
|
+
"""Finds the provider queryable corresponding to the given eodag key based on the metadata mapping
|
|
1529
|
+
|
|
1502
1530
|
:param eodag_key: key in eodag
|
|
1503
1531
|
:param provider_queryables: queryables returned from the provider
|
|
1504
1532
|
:param metadata_mapping: metadata mapping from which the keys are retrieved
|
eodag/cli.py
CHANGED
|
@@ -57,6 +57,11 @@ from eodag.utils import DEFAULT_ITEMS_PER_PAGE, DEFAULT_PAGE, parse_qs
|
|
|
57
57
|
from eodag.utils.exceptions import NoMatchingProductType, UnsupportedProvider
|
|
58
58
|
from eodag.utils.logging import setup_logging
|
|
59
59
|
|
|
60
|
+
try:
|
|
61
|
+
from eodag.rest.utils import LIVENESS_PROBE_PATH
|
|
62
|
+
except ImportError:
|
|
63
|
+
pass
|
|
64
|
+
|
|
60
65
|
if TYPE_CHECKING:
|
|
61
66
|
from click import Context
|
|
62
67
|
|
|
@@ -70,6 +75,18 @@ CRUNCHERS = [
|
|
|
70
75
|
]
|
|
71
76
|
|
|
72
77
|
|
|
78
|
+
class LivenessFilter:
|
|
79
|
+
"""
|
|
80
|
+
Filter out requests to the liveness probe endpoint
|
|
81
|
+
"""
|
|
82
|
+
|
|
83
|
+
def filter(self, record):
|
|
84
|
+
"""
|
|
85
|
+
Filter method required by the Python logging API.
|
|
86
|
+
"""
|
|
87
|
+
return LIVENESS_PROBE_PATH not in record.getMessage()
|
|
88
|
+
|
|
89
|
+
|
|
73
90
|
class MutuallyExclusiveOption(click.Option):
|
|
74
91
|
"""Mutually Exclusive Options for Click
|
|
75
92
|
from https://gist.github.com/jacobtolar/fb80d5552a9a9dfc32b12a829fa21c0c
|
|
@@ -679,7 +696,9 @@ def serve_rest(
|
|
|
679
696
|
try:
|
|
680
697
|
pid = os.fork()
|
|
681
698
|
except OSError as e:
|
|
682
|
-
raise Exception(
|
|
699
|
+
raise Exception(
|
|
700
|
+
"%s [%d]" % (e.strerror, e.errno) if e.errno is not None else e.strerror
|
|
701
|
+
)
|
|
683
702
|
|
|
684
703
|
if pid == 0:
|
|
685
704
|
os.setsid()
|
|
@@ -691,8 +710,10 @@ def serve_rest(
|
|
|
691
710
|
|
|
692
711
|
logging_config = uvicorn.config.LOGGING_CONFIG
|
|
693
712
|
uvicorn_fmt = "%(asctime)-15s %(name)-32s [%(levelname)-8s] %(message)s"
|
|
713
|
+
logging_config["filters"] = {"liveness": {"()": LivenessFilter}}
|
|
694
714
|
logging_config["formatters"]["access"]["fmt"] = uvicorn_fmt
|
|
695
715
|
logging_config["formatters"]["default"]["fmt"] = uvicorn_fmt
|
|
716
|
+
logging_config["loggers"]["uvicorn.access"]["filters"] = ["liveness"]
|
|
696
717
|
|
|
697
718
|
eodag_formatter = logging.Formatter(
|
|
698
719
|
"%(asctime)-15s %(name)-32s [%(levelname)-8s] (tid=%(thread)d) %(message)s"
|
eodag/config.py
CHANGED
|
@@ -287,8 +287,10 @@ class PluginConfig(yaml.YAMLObject):
|
|
|
287
287
|
#: Mapping for product type metadata (e.g. ``abstract``, ``licence``) which can be parsed from the provider
|
|
288
288
|
#: result
|
|
289
289
|
generic_product_type_parsable_metadata: Dict[str, str]
|
|
290
|
-
#: Mapping for product type properties which can be parsed from the result
|
|
290
|
+
#: Mapping for product type properties which can be parsed from the result and are not product type metadata
|
|
291
291
|
generic_product_type_parsable_properties: Dict[str, str]
|
|
292
|
+
#: Mapping for product type properties which cannot be parsed from the result and are not product type metadata
|
|
293
|
+
generic_product_type_unparsable_properties: Dict[str, str]
|
|
292
294
|
#: URL to fetch data for a single collection
|
|
293
295
|
single_collection_fetch_url: str
|
|
294
296
|
#: Query string to be added to the fetch_url to filter for a collection
|
|
@@ -307,6 +309,10 @@ class PluginConfig(yaml.YAMLObject):
|
|
|
307
309
|
result_type: str
|
|
308
310
|
#: JsonPath to retrieve the queryables from the provider result
|
|
309
311
|
results_entry: str
|
|
312
|
+
#: :class:`~eodag.plugins.search.base.Search` URL of the constraint file used to build queryables
|
|
313
|
+
constraints_url: str
|
|
314
|
+
#: :class:`~eodag.plugins.search.base.Search` Key in the json result where the constraints can be found
|
|
315
|
+
constraints_entry: str
|
|
310
316
|
|
|
311
317
|
class OrderOnResponse(TypedDict):
|
|
312
318
|
"""Configuration for order on-response during download"""
|
|
@@ -434,17 +440,6 @@ class PluginConfig(yaml.YAMLObject):
|
|
|
434
440
|
discover_queryables: PluginConfig.DiscoverQueryables
|
|
435
441
|
#: :class:`~eodag.plugins.search.base.Search` The mapping between eodag metadata and the plugin specific metadata
|
|
436
442
|
metadata_mapping: Dict[str, Union[str, List[str]]]
|
|
437
|
-
#: :class:`~eodag.plugins.search.base.Search` URL of the constraint file used to build queryables
|
|
438
|
-
constraints_file_url: str
|
|
439
|
-
#: :class:`~eodag.plugins.search.base.Search`
|
|
440
|
-
#: Key which is used in the eodag configuration to map the eodag product type to the provider product type
|
|
441
|
-
constraints_file_dataset_key: str
|
|
442
|
-
#: :class:`~eodag.plugins.search.base.Search` Key in the json result where the constraints can be found
|
|
443
|
-
constraints_entry: str
|
|
444
|
-
#: :class:`~eodag.plugins.search.base.Search`
|
|
445
|
-
#: Whether only a provider result containing constraints_entry is accepted as valid and used to create constraints
|
|
446
|
-
#: or not
|
|
447
|
-
stop_without_constraints_entry_key: bool
|
|
448
443
|
#: :class:`~eodag.plugins.search.base.Search` Parameters to remove from queryables
|
|
449
444
|
remove_from_queryables: List[str]
|
|
450
445
|
#: :class:`~eodag.plugins.search.base.Search` Parameters to be passed as is in the search url query string
|
|
@@ -477,15 +472,17 @@ class PluginConfig(yaml.YAMLObject):
|
|
|
477
472
|
#: :class:`~eodag.plugins.search.static_stac_search.StaticStacSearch`
|
|
478
473
|
#: Maximum number of connections for concurrent HTTP requests
|
|
479
474
|
max_connections: int
|
|
480
|
-
#: :class:`~eodag.plugins.search.build_search_result.
|
|
475
|
+
#: :class:`~eodag.plugins.search.build_search_result.ECMWFSearch`
|
|
481
476
|
#: Whether end date should be excluded from search request or not
|
|
482
477
|
end_date_excluded: bool
|
|
483
|
-
#: :class:`~eodag.plugins.search.build_search_result.
|
|
478
|
+
#: :class:`~eodag.plugins.search.build_search_result.ECMWFSearch`
|
|
484
479
|
#: List of parameters used to parse metadata but that must not be included to the query
|
|
485
480
|
remove_from_query: List[str]
|
|
486
481
|
#: :class:`~eodag.plugins.search.csw.CSWSearch`
|
|
487
482
|
#: OGC Catalogue Service version
|
|
488
483
|
version: str
|
|
484
|
+
#: :class:`~eodag.plugins.apis.ecmwf.EcmwfApi` url of the authentication endpoint
|
|
485
|
+
auth_endpoint: str
|
|
489
486
|
|
|
490
487
|
# download ---------------------------------------------------------------------------------------------------------
|
|
491
488
|
#: :class:`~eodag.plugins.download.base.Download` Default endpoint url
|
|
@@ -539,6 +536,9 @@ class PluginConfig(yaml.YAMLObject):
|
|
|
539
536
|
#: Dictionary containing all keys/value pairs that should be added to the headers
|
|
540
537
|
headers: Dict[str, str]
|
|
541
538
|
#: :class:`~eodag.plugins.authentication.base.Authentication`
|
|
539
|
+
#: Dictionary containing all keys/value pairs that should be added to the headers for token retrieve only
|
|
540
|
+
retrieve_headers: Dict[str, str]
|
|
541
|
+
#: :class:`~eodag.plugins.authentication.base.Authentication`
|
|
542
542
|
#: The key pointing to the token in the response from the token server
|
|
543
543
|
token_key: str
|
|
544
544
|
#: :class:`~eodag.plugins.authentication.base.Authentication`
|