eodag 3.0.0b2__py3-none-any.whl → 3.0.1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- eodag/__init__.py +6 -8
- eodag/api/core.py +295 -287
- eodag/api/product/__init__.py +10 -4
- eodag/api/product/_assets.py +2 -14
- eodag/api/product/_product.py +16 -30
- eodag/api/product/drivers/__init__.py +7 -2
- eodag/api/product/drivers/base.py +0 -3
- eodag/api/product/metadata_mapping.py +12 -31
- eodag/api/search_result.py +33 -12
- eodag/cli.py +35 -19
- eodag/config.py +455 -155
- eodag/plugins/apis/base.py +13 -7
- eodag/plugins/apis/ecmwf.py +16 -7
- eodag/plugins/apis/usgs.py +68 -16
- eodag/plugins/authentication/aws_auth.py +25 -7
- eodag/plugins/authentication/base.py +10 -1
- eodag/plugins/authentication/generic.py +14 -3
- eodag/plugins/authentication/header.py +12 -4
- eodag/plugins/authentication/keycloak.py +41 -22
- eodag/plugins/authentication/oauth.py +11 -1
- eodag/plugins/authentication/openid_connect.py +183 -167
- eodag/plugins/authentication/qsauth.py +12 -4
- eodag/plugins/authentication/sas_auth.py +19 -2
- eodag/plugins/authentication/token.py +59 -11
- eodag/plugins/authentication/token_exchange.py +19 -19
- eodag/plugins/crunch/base.py +7 -2
- eodag/plugins/crunch/filter_date.py +8 -11
- eodag/plugins/crunch/filter_latest_intersect.py +5 -7
- eodag/plugins/crunch/filter_latest_tpl_name.py +2 -5
- eodag/plugins/crunch/filter_overlap.py +9 -15
- eodag/plugins/crunch/filter_property.py +9 -14
- eodag/plugins/download/aws.py +84 -99
- eodag/plugins/download/base.py +36 -77
- eodag/plugins/download/creodias_s3.py +11 -2
- eodag/plugins/download/http.py +134 -109
- eodag/plugins/download/s3rest.py +37 -43
- eodag/plugins/manager.py +173 -41
- eodag/plugins/search/__init__.py +9 -9
- eodag/plugins/search/base.py +35 -35
- eodag/plugins/search/build_search_result.py +55 -64
- eodag/plugins/search/cop_marine.py +113 -32
- eodag/plugins/search/creodias_s3.py +20 -8
- eodag/plugins/search/csw.py +41 -1
- eodag/plugins/search/data_request_search.py +119 -14
- eodag/plugins/search/qssearch.py +619 -197
- eodag/plugins/search/static_stac_search.py +25 -23
- eodag/resources/ext_product_types.json +1 -1
- eodag/resources/product_types.yml +211 -56
- eodag/resources/providers.yml +1762 -1809
- eodag/resources/stac.yml +3 -163
- eodag/resources/user_conf_template.yml +134 -119
- eodag/rest/config.py +1 -2
- eodag/rest/constants.py +0 -1
- eodag/rest/core.py +70 -92
- eodag/rest/errors.py +181 -0
- eodag/rest/server.py +24 -330
- eodag/rest/stac.py +105 -630
- eodag/rest/types/eodag_search.py +17 -15
- eodag/rest/types/queryables.py +5 -14
- eodag/rest/types/stac_search.py +18 -13
- eodag/rest/utils/rfc3339.py +0 -1
- eodag/types/__init__.py +24 -6
- eodag/types/download_args.py +14 -5
- eodag/types/queryables.py +1 -2
- eodag/types/search_args.py +10 -11
- eodag/types/whoosh.py +0 -2
- eodag/utils/__init__.py +97 -136
- eodag/utils/constraints.py +0 -8
- eodag/utils/exceptions.py +23 -9
- eodag/utils/import_system.py +0 -4
- eodag/utils/logging.py +37 -80
- eodag/utils/notebook.py +4 -4
- eodag/utils/requests.py +13 -23
- eodag/utils/rest.py +0 -4
- eodag/utils/stac_reader.py +3 -15
- {eodag-3.0.0b2.dist-info → eodag-3.0.1.dist-info}/METADATA +41 -24
- eodag-3.0.1.dist-info/RECORD +109 -0
- {eodag-3.0.0b2.dist-info → eodag-3.0.1.dist-info}/WHEEL +1 -1
- {eodag-3.0.0b2.dist-info → eodag-3.0.1.dist-info}/entry_points.txt +1 -0
- eodag/resources/constraints/climate-dt.json +0 -13
- eodag/resources/constraints/extremes-dt.json +0 -8
- eodag-3.0.0b2.dist-info/RECORD +0 -110
- {eodag-3.0.0b2.dist-info → eodag-3.0.1.dist-info}/LICENSE +0 -0
- {eodag-3.0.0b2.dist-info → eodag-3.0.1.dist-info}/top_level.txt +0 -0
eodag/api/product/__init__.py
CHANGED
|
@@ -16,11 +16,17 @@
|
|
|
16
16
|
# See the License for the specific language governing permissions and
|
|
17
17
|
# limitations under the License.
|
|
18
18
|
#
|
|
19
|
-
# type: ignore
|
|
20
19
|
"""EODAG product package"""
|
|
21
20
|
try:
|
|
22
21
|
# import from eodag-cube if installed
|
|
23
|
-
from eodag_cube.api.product import
|
|
22
|
+
from eodag_cube.api.product import ( # pyright: ignore[reportMissingImports]
|
|
23
|
+
Asset,
|
|
24
|
+
AssetsDict,
|
|
25
|
+
EOProduct,
|
|
26
|
+
)
|
|
24
27
|
except ImportError:
|
|
25
|
-
from ._assets import Asset, AssetsDict #
|
|
26
|
-
from ._product import EOProduct #
|
|
28
|
+
from ._assets import Asset, AssetsDict # type: ignore[assignment]
|
|
29
|
+
from ._product import EOProduct # type: ignore[assignment]
|
|
30
|
+
|
|
31
|
+
# exportable content
|
|
32
|
+
__all__ = ["Asset", "AssetsDict", "EOProduct"]
|
eodag/api/product/_assets.py
CHANGED
|
@@ -35,11 +35,8 @@ class AssetsDict(UserDict):
|
|
|
35
35
|
:class:`~eodag.api.product._product.EOProduct` resulting from a search.
|
|
36
36
|
|
|
37
37
|
:param product: Product resulting from a search
|
|
38
|
-
:type product: :class:`~eodag.api.product._product.EOProduct`
|
|
39
38
|
:param args: (optional) Arguments used to init the dictionary
|
|
40
|
-
:type args: Any
|
|
41
39
|
:param kwargs: (optional) Additional named-arguments used to init the dictionary
|
|
42
|
-
:type kwargs: Any
|
|
43
40
|
"""
|
|
44
41
|
|
|
45
42
|
product: EOProduct
|
|
@@ -56,17 +53,15 @@ class AssetsDict(UserDict):
|
|
|
56
53
|
|
|
57
54
|
:returns: The representation of a :class:`~eodag.api.product._assets.AssetsDict`
|
|
58
55
|
as a Python dict
|
|
59
|
-
:rtype: dict
|
|
60
56
|
"""
|
|
61
57
|
return {k: v.as_dict() for k, v in self.data.items()}
|
|
62
58
|
|
|
63
59
|
def get_values(self, asset_filter: str = "") -> List[Asset]:
|
|
64
60
|
"""
|
|
65
61
|
retrieves the assets matching the given filter
|
|
66
|
-
|
|
67
|
-
:
|
|
62
|
+
|
|
63
|
+
:param asset_filter: regex filter with which the assets should be matched
|
|
68
64
|
:return: list of assets
|
|
69
|
-
:rtype: List[Asset]
|
|
70
65
|
"""
|
|
71
66
|
if asset_filter:
|
|
72
67
|
filter_regex = re.compile(asset_filter)
|
|
@@ -128,13 +123,9 @@ class Asset(UserDict):
|
|
|
128
123
|
:class:`~eodag.api.product._product.EOProduct` resulting from a search.
|
|
129
124
|
|
|
130
125
|
:param product: Product resulting from a search
|
|
131
|
-
:type product: :class:`~eodag.api.product._product.EOProduct`
|
|
132
126
|
:param key: asset key
|
|
133
|
-
:type key: str
|
|
134
127
|
:param args: (optional) Arguments used to init the dictionary
|
|
135
|
-
:type args: Any
|
|
136
128
|
:param kwargs: (optional) Additional named-arguments used to init the dictionary
|
|
137
|
-
:type kwargs: Any
|
|
138
129
|
"""
|
|
139
130
|
|
|
140
131
|
product: EOProduct
|
|
@@ -152,7 +143,6 @@ class Asset(UserDict):
|
|
|
152
143
|
|
|
153
144
|
:returns: The representation of a :class:`~eodag.api.product._assets.Asset` as a
|
|
154
145
|
Python dict
|
|
155
|
-
:rtype: dict
|
|
156
146
|
"""
|
|
157
147
|
return self.data
|
|
158
148
|
|
|
@@ -160,9 +150,7 @@ class Asset(UserDict):
|
|
|
160
150
|
"""Downloads a single asset
|
|
161
151
|
|
|
162
152
|
:param kwargs: (optional) Additional named-arguments passed to `plugin.download()`
|
|
163
|
-
:type kwargs: Any
|
|
164
153
|
:returns: The absolute path to the downloaded product on the local filesystem
|
|
165
|
-
:rtype: str
|
|
166
154
|
"""
|
|
167
155
|
return self.product.download(asset=self.key, **kwargs)
|
|
168
156
|
|
eodag/api/product/_product.py
CHANGED
|
@@ -32,9 +32,11 @@ from shapely.errors import ShapelyError
|
|
|
32
32
|
|
|
33
33
|
try:
|
|
34
34
|
# import from eodag-cube if installed
|
|
35
|
-
from eodag_cube.api.product import
|
|
35
|
+
from eodag_cube.api.product import ( # pyright: ignore[reportMissingImports]
|
|
36
|
+
AssetsDict,
|
|
37
|
+
)
|
|
36
38
|
except ImportError:
|
|
37
|
-
from eodag.api.product._assets import AssetsDict
|
|
39
|
+
from eodag.api.product._assets import AssetsDict
|
|
38
40
|
|
|
39
41
|
from eodag.api.product.drivers import DRIVERS, NoDriver
|
|
40
42
|
from eodag.api.product.metadata_mapping import (
|
|
@@ -86,9 +88,7 @@ class EOProduct:
|
|
|
86
88
|
parameters that led to its creation.
|
|
87
89
|
|
|
88
90
|
:param provider: The provider from which the product originates
|
|
89
|
-
:type provider: str
|
|
90
91
|
:param properties: The metadata of the product
|
|
91
|
-
:type properties: dict
|
|
92
92
|
:ivar product_type: The product type
|
|
93
93
|
:vartype product_type: str
|
|
94
94
|
:ivar location: The path to the product, either remote or local if downloaded
|
|
@@ -180,7 +180,6 @@ class EOProduct:
|
|
|
180
180
|
|
|
181
181
|
:returns: The representation of a :class:`~eodag.api.product._product.EOProduct` as a
|
|
182
182
|
Python dict
|
|
183
|
-
:rtype: dict
|
|
184
183
|
"""
|
|
185
184
|
search_intersection = None
|
|
186
185
|
if self.search_intersection is not None:
|
|
@@ -212,9 +211,7 @@ class EOProduct:
|
|
|
212
211
|
|
|
213
212
|
:param feature: The representation of a :class:`~eodag.api.product._product.EOProduct`
|
|
214
213
|
as a Python dict
|
|
215
|
-
:type feature: dict
|
|
216
214
|
:returns: An instance of :class:`~eodag.api.product._product.EOProduct`
|
|
217
|
-
:rtype: :class:`~eodag.api.product._product.EOProduct`
|
|
218
215
|
"""
|
|
219
216
|
properties = feature["properties"]
|
|
220
217
|
properties["geometry"] = feature["geometry"]
|
|
@@ -248,11 +245,9 @@ class EOProduct:
|
|
|
248
245
|
"""Give to the product the information needed to download itself.
|
|
249
246
|
|
|
250
247
|
:param downloader: The download method that it can use
|
|
251
|
-
:type downloader: Concrete subclass of
|
|
252
248
|
:class:`~eodag.plugins.download.base.Download` or
|
|
253
249
|
:class:`~eodag.plugins.api.base.Api`
|
|
254
250
|
:param authenticator: The authentication method needed to perform the download
|
|
255
|
-
:type authenticator: Concrete subclass of
|
|
256
251
|
:class:`~eodag.plugins.authentication.base.Authentication`
|
|
257
252
|
"""
|
|
258
253
|
self.downloader = downloader
|
|
@@ -302,20 +297,15 @@ class EOProduct:
|
|
|
302
297
|
size as inputs and handle progress bar
|
|
303
298
|
creation and update to give the user a
|
|
304
299
|
feedback on the download progress
|
|
305
|
-
:type progress_callback: :class:`~eodag.utils.ProgressCallback` or None
|
|
306
300
|
:param wait: (optional) If download fails, wait time in minutes between
|
|
307
301
|
two download tries
|
|
308
|
-
:type wait: int
|
|
309
302
|
:param timeout: (optional) If download fails, maximum time in minutes
|
|
310
303
|
before stop retrying to download
|
|
311
|
-
:
|
|
312
|
-
:param kwargs: `outputs_prefix` (str), `extract` (bool), `delete_archive` (bool)
|
|
304
|
+
:param kwargs: `output_dir` (str), `extract` (bool), `delete_archive` (bool)
|
|
313
305
|
and `dl_url_params` (dict) can be provided as additional kwargs
|
|
314
306
|
and will override any other values defined in a configuration
|
|
315
307
|
file or with environment variables.
|
|
316
|
-
:type kwargs: Union[str, bool, dict]
|
|
317
308
|
:returns: The absolute path to the downloaded product on the local filesystem
|
|
318
|
-
:rtype: str
|
|
319
309
|
:raises: :class:`~eodag.utils.exceptions.PluginImplementationError`
|
|
320
310
|
:raises: :class:`RuntimeError`
|
|
321
311
|
"""
|
|
@@ -383,7 +373,7 @@ class EOProduct:
|
|
|
383
373
|
def get_quicklook(
|
|
384
374
|
self,
|
|
385
375
|
filename: Optional[str] = None,
|
|
386
|
-
|
|
376
|
+
output_dir: Optional[str] = None,
|
|
387
377
|
progress_callback: Optional[ProgressCallback] = None,
|
|
388
378
|
) -> str:
|
|
389
379
|
"""Download the quicklook image of a given EOProduct from its provider if it
|
|
@@ -391,18 +381,14 @@ class EOProduct:
|
|
|
391
381
|
|
|
392
382
|
:param filename: (optional) The name to give to the downloaded quicklook. If not
|
|
393
383
|
given, it defaults to the product's ID (without file extension).
|
|
394
|
-
:
|
|
395
|
-
:param base_dir: (optional) The absolute path of the directory where to store
|
|
384
|
+
:param output_dir: (optional) The absolute path of the directory where to store
|
|
396
385
|
the quicklooks in the filesystem. If not given, it defaults to the
|
|
397
|
-
`quicklooks` directory under this EO product downloader's ``
|
|
386
|
+
`quicklooks` directory under this EO product downloader's ``output_dir``
|
|
398
387
|
config param (e.g. '/tmp/quicklooks/')
|
|
399
|
-
:type base_dir: str
|
|
400
388
|
:param progress_callback: (optional) A method or a callable object which takes
|
|
401
389
|
a current size and a maximum size as inputs and handle progress bar
|
|
402
390
|
creation and update to give the user a feedback on the download progress
|
|
403
|
-
:type progress_callback: :class:`~eodag.utils.ProgressCallback` or None
|
|
404
391
|
:returns: The absolute path of the downloaded quicklook
|
|
405
|
-
:rtype: str
|
|
406
392
|
"""
|
|
407
393
|
|
|
408
394
|
def format_quicklook_address() -> None:
|
|
@@ -439,20 +425,20 @@ class EOProduct:
|
|
|
439
425
|
|
|
440
426
|
format_quicklook_address()
|
|
441
427
|
|
|
442
|
-
if
|
|
443
|
-
|
|
428
|
+
if output_dir is not None:
|
|
429
|
+
quicklooks_output_dir = os.path.abspath(os.path.realpath(output_dir))
|
|
444
430
|
else:
|
|
445
431
|
tempdir = tempfile.gettempdir()
|
|
446
|
-
|
|
447
|
-
getattr(self.downloader.config, "
|
|
432
|
+
downloader_output_dir = (
|
|
433
|
+
getattr(self.downloader.config, "output_dir", tempdir)
|
|
448
434
|
if self.downloader
|
|
449
435
|
else tempdir
|
|
450
436
|
)
|
|
451
|
-
|
|
452
|
-
if not os.path.isdir(
|
|
453
|
-
os.makedirs(
|
|
437
|
+
quicklooks_output_dir = os.path.join(downloader_output_dir, "quicklooks")
|
|
438
|
+
if not os.path.isdir(quicklooks_output_dir):
|
|
439
|
+
os.makedirs(quicklooks_output_dir)
|
|
454
440
|
quicklook_file = os.path.join(
|
|
455
|
-
|
|
441
|
+
quicklooks_output_dir,
|
|
456
442
|
filename if filename is not None else self.properties["id"],
|
|
457
443
|
)
|
|
458
444
|
|
|
@@ -16,9 +16,14 @@
|
|
|
16
16
|
# See the License for the specific language governing permissions and
|
|
17
17
|
# limitations under the License.
|
|
18
18
|
"""EODAG drivers package"""
|
|
19
|
-
from eodag.api.product.drivers.base import DatasetDriver, NoDriver
|
|
19
|
+
from eodag.api.product.drivers.base import DatasetDriver, NoDriver
|
|
20
20
|
|
|
21
21
|
try:
|
|
22
|
-
from eodag_cube.api.product.drivers import
|
|
22
|
+
from eodag_cube.api.product.drivers import ( # pyright: ignore[reportMissingImports]
|
|
23
|
+
DRIVERS,
|
|
24
|
+
)
|
|
23
25
|
except ImportError:
|
|
24
26
|
DRIVERS = []
|
|
27
|
+
|
|
28
|
+
# exportable content
|
|
29
|
+
__all__ = ["DRIVERS", "DatasetDriver", "NoDriver"]
|
|
@@ -30,11 +30,8 @@ class DatasetDriver(metaclass=type):
|
|
|
30
30
|
"""Retrieve the address of the dataset represented by `eo_product`.
|
|
31
31
|
|
|
32
32
|
:param eo_product: The product whom underlying dataset address is to be retrieved
|
|
33
|
-
:type eo_product: :class:`~eodag.api.product._product.EOProduct`
|
|
34
33
|
:param band: The band to retrieve (e.g: 'B01')
|
|
35
|
-
:type band: str
|
|
36
34
|
:returns: An address for the dataset
|
|
37
|
-
:rtype: str
|
|
38
35
|
:raises: :class:`~eodag.utils.exceptions.AddressNotFound`
|
|
39
36
|
:raises: :class:`~eodag.utils.exceptions.UnsupportedDatasetAddressScheme`
|
|
40
37
|
"""
|
|
@@ -41,6 +41,7 @@ import geojson
|
|
|
41
41
|
import orjson
|
|
42
42
|
import pyproj
|
|
43
43
|
from dateutil.parser import isoparse
|
|
44
|
+
from dateutil.relativedelta import relativedelta
|
|
44
45
|
from dateutil.tz import UTC, tzutc
|
|
45
46
|
from jsonpath_ng.jsonpath import Child, JSONPath
|
|
46
47
|
from lxml import etree
|
|
@@ -120,10 +121,8 @@ def get_metadata_path(
|
|
|
120
121
|
in the provider search config. For example, it is the list
|
|
121
122
|
`['productType', '$.properties.productType']` with the sample
|
|
122
123
|
above. Or the string `$.properties.id`.
|
|
123
|
-
:type map_value: str or list(str)
|
|
124
124
|
:returns: Either, None and the path to the metadata value, or a list of converter
|
|
125
125
|
and its args, and the path to the metadata value.
|
|
126
|
-
:rtype: tuple(list(str) or None, str)
|
|
127
126
|
"""
|
|
128
127
|
path = get_metadata_path_value(map_value)
|
|
129
128
|
try:
|
|
@@ -147,16 +146,14 @@ def get_search_param(map_value: List[str]) -> str:
|
|
|
147
146
|
|
|
148
147
|
:param map_value: The value originating from the definition of `metadata_mapping`
|
|
149
148
|
in the provider search config
|
|
150
|
-
:type map_value: list
|
|
151
149
|
:returns: The value of the search parameter as defined in the provider config
|
|
152
|
-
:rtype: str
|
|
153
150
|
"""
|
|
154
151
|
# Assume that caller will pass in the value as a list
|
|
155
152
|
return map_value[0]
|
|
156
153
|
|
|
157
154
|
|
|
158
155
|
def format_metadata(search_param: str, *args: Any, **kwargs: Any) -> str:
|
|
159
|
-
"""Format a string of form {<field_name>#<conversion_function>}
|
|
156
|
+
"""Format a string of form ``{<field_name>#<conversion_function>}``
|
|
160
157
|
|
|
161
158
|
The currently understood converters are:
|
|
162
159
|
- ``datetime_to_timestamp_milliseconds``: converts a utc date string to a timestamp in
|
|
@@ -190,13 +187,9 @@ def format_metadata(search_param: str, *args: Any, **kwargs: Any) -> str:
|
|
|
190
187
|
- ``get_ecmwf_time``: get the time of a datetime string in the ECMWF format
|
|
191
188
|
|
|
192
189
|
:param search_param: The string to be formatted
|
|
193
|
-
:type search_param: str
|
|
194
190
|
:param args: (optional) Additional arguments to use in the formatting process
|
|
195
|
-
:type args: tuple
|
|
196
191
|
:param kwargs: (optional) Additional named-arguments to use when formatting
|
|
197
|
-
:type kwargs: Any
|
|
198
192
|
:returns: The formatted string
|
|
199
|
-
:rtype: str
|
|
200
193
|
"""
|
|
201
194
|
|
|
202
195
|
class MetadataFormatter(Formatter):
|
|
@@ -839,7 +832,7 @@ def format_metadata(search_param: str, *args: Any, **kwargs: Any) -> str:
|
|
|
839
832
|
def convert_get_hydrological_year(date: str):
|
|
840
833
|
utc_date = MetadataFormatter.convert_to_iso_utc_datetime(date)
|
|
841
834
|
date_object = datetime.strptime(utc_date, "%Y-%m-%dT%H:%M:%S.%fZ")
|
|
842
|
-
date_object_second_year = date_object +
|
|
835
|
+
date_object_second_year = date_object + relativedelta(years=1)
|
|
843
836
|
return [
|
|
844
837
|
f'{date_object.strftime("%Y")}_{date_object_second_year.strftime("%y")}'
|
|
845
838
|
]
|
|
@@ -926,7 +919,6 @@ def properties_from_json(
|
|
|
926
919
|
"""Extract properties from a provider json result.
|
|
927
920
|
|
|
928
921
|
:param json: The representation of a provider result as a json object
|
|
929
|
-
:type json: dict
|
|
930
922
|
:param mapping: A mapping between :class:`~eodag.api.product._product.EOProduct`'s metadata
|
|
931
923
|
keys and the location of the values of these properties in the json
|
|
932
924
|
representation, expressed as a
|
|
@@ -934,9 +926,7 @@ def properties_from_json(
|
|
|
934
926
|
:param discovery_config: (optional) metadata discovery configuration dict, accepting among other items
|
|
935
927
|
`discovery_pattern` (Regex pattern for metadata key discovery, e.g. "^[a-zA-Z]+$"),
|
|
936
928
|
`discovery_path` (String representation of jsonpath)
|
|
937
|
-
:type discovery_config: dict
|
|
938
929
|
:returns: The metadata of the :class:`~eodag.api.product._product.EOProduct`
|
|
939
|
-
:rtype: dict
|
|
940
930
|
"""
|
|
941
931
|
properties: Dict[str, Any] = {}
|
|
942
932
|
templates = {}
|
|
@@ -1073,7 +1063,6 @@ def properties_from_xml(
|
|
|
1073
1063
|
"""Extract properties from a provider xml result.
|
|
1074
1064
|
|
|
1075
1065
|
:param xml_as_text: The representation of a provider result as xml
|
|
1076
|
-
:type xml_as_text: str
|
|
1077
1066
|
:param mapping: A mapping between :class:`~eodag.api.product._product.EOProduct`'s metadata
|
|
1078
1067
|
keys and the location of the values of these properties in the xml
|
|
1079
1068
|
representation, expressed as a
|
|
@@ -1083,13 +1072,10 @@ def properties_from_xml(
|
|
|
1083
1072
|
not supporting empty namespace prefix. The
|
|
1084
1073
|
xpath in `mapping` must use this value to be able to
|
|
1085
1074
|
correctly reach empty-namespace prefixed elements
|
|
1086
|
-
:type empty_ns_prefix: str
|
|
1087
1075
|
:param discovery_config: (optional) metadata discovery configuration dict, accepting among other items
|
|
1088
1076
|
`discovery_pattern` (Regex pattern for metadata key discovery, e.g. "^[a-zA-Z]+$"),
|
|
1089
1077
|
`discovery_path` (String representation of xpath)
|
|
1090
|
-
:type discovery_config: dict
|
|
1091
1078
|
:returns: the metadata of the :class:`~eodag.api.product._product.EOProduct`
|
|
1092
|
-
:rtype: dict
|
|
1093
1079
|
"""
|
|
1094
1080
|
properties: Dict[str, Any] = {}
|
|
1095
1081
|
templates = {}
|
|
@@ -1223,16 +1209,13 @@ def mtd_cfg_as_conversion_and_querypath(
|
|
|
1223
1209
|
dest_dict: Dict[str, Any] = {},
|
|
1224
1210
|
result_type: str = "json",
|
|
1225
1211
|
) -> Dict[str, Any]:
|
|
1226
|
-
"""Metadata configuration dictionary to querypath with conversion
|
|
1212
|
+
"""Metadata configuration dictionary to querypath with conversion dictionary
|
|
1227
1213
|
Transform every src_dict value from jsonpath_str to tuple `(conversion, jsonpath_object)`
|
|
1228
1214
|
or from xpath_str to tuple `(conversion, xpath_str)`
|
|
1229
1215
|
|
|
1230
1216
|
:param src_dict: Input dict containing jsonpath str as values
|
|
1231
|
-
:type src_dict: dict
|
|
1232
1217
|
:param dest_dict: (optional) Output dict containing jsonpath objects as values
|
|
1233
|
-
:type dest_dict: dict
|
|
1234
1218
|
:returns: dest_dict
|
|
1235
|
-
:rtype: dict
|
|
1236
1219
|
"""
|
|
1237
1220
|
# check if the configuration has already been converted
|
|
1238
1221
|
some_configured_value = (
|
|
@@ -1476,13 +1459,18 @@ def get_queryable_from_provider(
|
|
|
1476
1459
|
"""Get EODAG configured queryable parameter from provider queryable parameter
|
|
1477
1460
|
|
|
1478
1461
|
:param provider_queryable: provider queryable parameter
|
|
1479
|
-
:type provider_queryable: str
|
|
1480
1462
|
:param metadata_mapping: metadata-mapping configuration
|
|
1481
|
-
:type metadata_mapping: Dict[str, Union[str, List[str]]])
|
|
1482
1463
|
:returns: EODAG configured queryable parameter or None
|
|
1483
|
-
:rtype: Optional[str]
|
|
1484
1464
|
"""
|
|
1485
1465
|
pattern = rf"\b{provider_queryable}\b"
|
|
1466
|
+
# if 1:1 mapping exists privilege this one instead of other mapping
|
|
1467
|
+
# e.g. provider queryable = year -> use year and not date in which year also appears
|
|
1468
|
+
mapping_values = [
|
|
1469
|
+
v[0] if isinstance(v, list) else "" for v in metadata_mapping.values()
|
|
1470
|
+
]
|
|
1471
|
+
if provider_queryable in mapping_values:
|
|
1472
|
+
ind = mapping_values.index(provider_queryable)
|
|
1473
|
+
return Queryables.get_queryable_from_alias(list(metadata_mapping.keys())[ind])
|
|
1486
1474
|
for param, param_conf in metadata_mapping.items():
|
|
1487
1475
|
if isinstance(param_conf, list) and re.search(pattern, param_conf[0]):
|
|
1488
1476
|
return Queryables.get_queryable_from_alias(param)
|
|
@@ -1495,11 +1483,8 @@ def get_provider_queryable_path(
|
|
|
1495
1483
|
"""Get EODAG configured queryable path from its parameter
|
|
1496
1484
|
|
|
1497
1485
|
:param queryable: eodag queryable parameter
|
|
1498
|
-
:type queryable: str
|
|
1499
1486
|
:param metadata_mapping: metadata-mapping configuration
|
|
1500
|
-
:type metadata_mapping: Dict[str, Union[str, List[str]]])
|
|
1501
1487
|
:returns: EODAG configured queryable path or None
|
|
1502
|
-
:rtype: Optional[str]
|
|
1503
1488
|
"""
|
|
1504
1489
|
parameter_conf = metadata_mapping.get(queryable, None)
|
|
1505
1490
|
if isinstance(parameter_conf, list):
|
|
@@ -1515,13 +1500,9 @@ def get_provider_queryable_key(
|
|
|
1515
1500
|
) -> str:
|
|
1516
1501
|
"""finds the provider queryable corresponding to the given eodag key based on the metadata mapping
|
|
1517
1502
|
:param eodag_key: key in eodag
|
|
1518
|
-
:type eodag_key: str
|
|
1519
1503
|
:param provider_queryables: queryables returned from the provider
|
|
1520
|
-
:type provider_queryables: dict
|
|
1521
1504
|
:param metadata_mapping: metadata mapping from which the keys are retrieved
|
|
1522
|
-
:type metadata_mapping: Dict[str, Union[List[Any], str]]
|
|
1523
1505
|
:returns: provider queryable key
|
|
1524
|
-
:rtype: str
|
|
1525
1506
|
"""
|
|
1526
1507
|
if eodag_key not in metadata_mapping:
|
|
1527
1508
|
return ""
|
eodag/api/search_result.py
CHANGED
|
@@ -18,9 +18,20 @@
|
|
|
18
18
|
from __future__ import annotations
|
|
19
19
|
|
|
20
20
|
from collections import UserList
|
|
21
|
-
from typing import
|
|
21
|
+
from typing import (
|
|
22
|
+
TYPE_CHECKING,
|
|
23
|
+
Annotated,
|
|
24
|
+
Any,
|
|
25
|
+
Dict,
|
|
26
|
+
Iterable,
|
|
27
|
+
List,
|
|
28
|
+
Optional,
|
|
29
|
+
Tuple,
|
|
30
|
+
Union,
|
|
31
|
+
)
|
|
22
32
|
|
|
23
33
|
from shapely.geometry import GeometryCollection, shape
|
|
34
|
+
from typing_extensions import Doc
|
|
24
35
|
|
|
25
36
|
from eodag.api.product import EOProduct
|
|
26
37
|
from eodag.plugins.crunch.filter_date import FilterDate
|
|
@@ -39,28 +50,34 @@ class SearchResult(UserList):
|
|
|
39
50
|
"""An object representing a collection of :class:`~eodag.api.product._product.EOProduct` resulting from a search.
|
|
40
51
|
|
|
41
52
|
:param products: A list of products resulting from a search
|
|
42
|
-
:type products: list(:class:`~eodag.api.product._product.EOProduct`)
|
|
43
53
|
:param number_matched: (optional) the estimated total number of matching results
|
|
44
|
-
|
|
54
|
+
|
|
55
|
+
:cvar data: List of products
|
|
56
|
+
:ivar number_matched: Estimated total number of matching results
|
|
45
57
|
"""
|
|
46
58
|
|
|
47
59
|
data: List[EOProduct]
|
|
48
60
|
|
|
61
|
+
errors: Annotated[
|
|
62
|
+
List[Tuple[str, Exception]], Doc("Tuple of provider name, exception")
|
|
63
|
+
]
|
|
64
|
+
|
|
49
65
|
def __init__(
|
|
50
|
-
self,
|
|
66
|
+
self,
|
|
67
|
+
products: List[EOProduct],
|
|
68
|
+
number_matched: Optional[int] = None,
|
|
69
|
+
errors: List[Tuple[str, Exception]] = [],
|
|
51
70
|
) -> None:
|
|
52
|
-
super(
|
|
71
|
+
super().__init__(products)
|
|
53
72
|
self.number_matched = number_matched
|
|
73
|
+
self.errors = errors
|
|
54
74
|
|
|
55
75
|
def crunch(self, cruncher: Crunch, **search_params: Any) -> SearchResult:
|
|
56
76
|
"""Do some crunching with the underlying EO products.
|
|
57
77
|
|
|
58
78
|
:param cruncher: The plugin instance to use to work on the products
|
|
59
|
-
:type cruncher: subclass of :class:`~eodag.plugins.crunch.base.Crunch`
|
|
60
79
|
:param search_params: The criteria that have been used to produce this result
|
|
61
|
-
:type search_params: dict
|
|
62
80
|
:returns: The result of the application of the crunching method to the EO products
|
|
63
|
-
:rtype: :class:`~eodag.api.search_result.SearchResult`
|
|
64
81
|
"""
|
|
65
82
|
crunched_results = cruncher.proceed(self.data, **search_params)
|
|
66
83
|
return SearchResult(crunched_results)
|
|
@@ -135,9 +152,7 @@ class SearchResult(UserList):
|
|
|
135
152
|
"""Builds an :class:`~eodag.api.search_result.SearchResult` object from its representation as geojson
|
|
136
153
|
|
|
137
154
|
:param feature_collection: A collection representing a search result.
|
|
138
|
-
:type feature_collection: dict
|
|
139
155
|
:returns: An eodag representation of a search result
|
|
140
|
-
:rtype: :class:`~eodag.api.search_result.SearchResult`
|
|
141
156
|
"""
|
|
142
157
|
return SearchResult(
|
|
143
158
|
[
|
|
@@ -154,7 +169,7 @@ class SearchResult(UserList):
|
|
|
154
169
|
}
|
|
155
170
|
|
|
156
171
|
def as_shapely_geometry_object(self) -> GeometryCollection:
|
|
157
|
-
""":class:`shapely.
|
|
172
|
+
""":class:`shapely.GeometryCollection` representation of SearchResult"""
|
|
158
173
|
return GeometryCollection(
|
|
159
174
|
[
|
|
160
175
|
shape(feature["geometry"]).buffer(0)
|
|
@@ -201,12 +216,18 @@ class SearchResult(UserList):
|
|
|
201
216
|
+ "</table>"
|
|
202
217
|
)
|
|
203
218
|
|
|
219
|
+
def extend(self, other: Iterable) -> None:
|
|
220
|
+
"""override extend method to include errors"""
|
|
221
|
+
if isinstance(other, SearchResult):
|
|
222
|
+
self.errors.extend(other.errors)
|
|
223
|
+
|
|
224
|
+
return super().extend(other)
|
|
225
|
+
|
|
204
226
|
|
|
205
227
|
class RawSearchResult(UserList):
|
|
206
228
|
"""An object representing a collection of raw/unparsed search results obtained from a provider.
|
|
207
229
|
|
|
208
230
|
:param results: A list of raw/unparsed search results
|
|
209
|
-
:type results: List[Any]
|
|
210
231
|
"""
|
|
211
232
|
|
|
212
233
|
data: List[Any]
|
eodag/cli.py
CHANGED
|
@@ -39,6 +39,7 @@ Commands:
|
|
|
39
39
|
|
|
40
40
|
noqa: D103
|
|
41
41
|
"""
|
|
42
|
+
|
|
42
43
|
from __future__ import annotations
|
|
43
44
|
|
|
44
45
|
import json
|
|
@@ -553,12 +554,13 @@ def download(ctx: Context, **kwargs: Any) -> None:
|
|
|
553
554
|
|
|
554
555
|
for idx, product in enumerate(search_results):
|
|
555
556
|
if product.downloader is None:
|
|
557
|
+
downloader = satim_api._plugins_manager.get_download_plugin(product)
|
|
556
558
|
auth = product.downloader_auth
|
|
557
559
|
if auth is None:
|
|
558
|
-
auth = satim_api._plugins_manager.get_auth_plugin(
|
|
559
|
-
|
|
560
|
-
|
|
561
|
-
)
|
|
560
|
+
auth = satim_api._plugins_manager.get_auth_plugin(
|
|
561
|
+
downloader, product
|
|
562
|
+
)
|
|
563
|
+
search_results[idx].register_downloader(downloader, auth)
|
|
562
564
|
|
|
563
565
|
downloaded_file = product.get_quicklook()
|
|
564
566
|
if not downloaded_file:
|
|
@@ -573,12 +575,13 @@ def download(ctx: Context, **kwargs: Any) -> None:
|
|
|
573
575
|
# register downloader
|
|
574
576
|
for idx, product in enumerate(search_results):
|
|
575
577
|
if product.downloader is None:
|
|
578
|
+
downloader = satim_api._plugins_manager.get_download_plugin(product)
|
|
576
579
|
auth = product.downloader_auth
|
|
577
580
|
if auth is None:
|
|
578
|
-
auth = satim_api._plugins_manager.get_auth_plugin(
|
|
579
|
-
|
|
580
|
-
|
|
581
|
-
)
|
|
581
|
+
auth = satim_api._plugins_manager.get_auth_plugin(
|
|
582
|
+
downloader, product
|
|
583
|
+
)
|
|
584
|
+
search_results[idx].register_downloader(downloader, auth)
|
|
582
585
|
|
|
583
586
|
downloaded_files = satim_api.download_all(search_results)
|
|
584
587
|
if downloaded_files and len(downloaded_files) > 0:
|
|
@@ -684,19 +687,32 @@ def serve_rest(
|
|
|
684
687
|
else:
|
|
685
688
|
sys.exit(0)
|
|
686
689
|
else:
|
|
690
|
+
import logging
|
|
691
|
+
|
|
687
692
|
logging_config = uvicorn.config.LOGGING_CONFIG
|
|
688
|
-
|
|
689
|
-
|
|
690
|
-
|
|
691
|
-
|
|
692
|
-
|
|
693
|
-
|
|
694
|
-
|
|
695
|
-
|
|
696
|
-
|
|
697
|
-
|
|
698
|
-
|
|
693
|
+
uvicorn_fmt = "%(asctime)-15s %(name)-32s [%(levelname)-8s] %(message)s"
|
|
694
|
+
logging_config["formatters"]["access"]["fmt"] = uvicorn_fmt
|
|
695
|
+
logging_config["formatters"]["default"]["fmt"] = uvicorn_fmt
|
|
696
|
+
|
|
697
|
+
eodag_formatter = logging.Formatter(
|
|
698
|
+
"%(asctime)-15s %(name)-32s [%(levelname)-8s] (tid=%(thread)d) %(message)s"
|
|
699
|
+
)
|
|
700
|
+
logging.getLogger("eodag").handlers[0].setFormatter(eodag_formatter)
|
|
701
|
+
|
|
702
|
+
if ctx.obj["verbosity"] <= 1:
|
|
703
|
+
logging_config["handlers"]["null"] = {
|
|
704
|
+
"level": "DEBUG",
|
|
705
|
+
"class": "logging.NullHandler",
|
|
699
706
|
}
|
|
707
|
+
logging_config["loggers"]["uvicorn"]["handlers"] = ["null"]
|
|
708
|
+
logging_config["loggers"]["uvicorn.error"]["handlers"] = ["null"]
|
|
709
|
+
logging_config["loggers"]["uvicorn.access"]["handlers"] = ["null"]
|
|
710
|
+
else:
|
|
711
|
+
log_level = "INFO" if ctx.obj["verbosity"] == 2 else "DEBUG"
|
|
712
|
+
logging_config["loggers"]["uvicorn"]["level"] = log_level
|
|
713
|
+
logging_config["loggers"]["uvicorn.error"]["level"] = log_level
|
|
714
|
+
logging_config["loggers"]["uvicorn.access"]["level"] = log_level
|
|
715
|
+
|
|
700
716
|
uvicorn.run(
|
|
701
717
|
"eodag.rest.server:app",
|
|
702
718
|
host=bind_host,
|