eodag 3.0.0b3__py3-none-any.whl → 3.0.1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- eodag/api/core.py +189 -125
- eodag/api/product/metadata_mapping.py +12 -3
- eodag/api/search_result.py +29 -3
- eodag/cli.py +35 -19
- eodag/config.py +412 -116
- eodag/plugins/apis/base.py +10 -4
- eodag/plugins/apis/ecmwf.py +14 -4
- eodag/plugins/apis/usgs.py +25 -2
- eodag/plugins/authentication/aws_auth.py +14 -5
- eodag/plugins/authentication/base.py +10 -1
- eodag/plugins/authentication/generic.py +14 -3
- eodag/plugins/authentication/header.py +12 -4
- eodag/plugins/authentication/keycloak.py +41 -22
- eodag/plugins/authentication/oauth.py +11 -1
- eodag/plugins/authentication/openid_connect.py +178 -163
- eodag/plugins/authentication/qsauth.py +12 -4
- eodag/plugins/authentication/sas_auth.py +19 -2
- eodag/plugins/authentication/token.py +57 -10
- eodag/plugins/authentication/token_exchange.py +19 -19
- eodag/plugins/crunch/base.py +4 -1
- eodag/plugins/crunch/filter_date.py +5 -2
- eodag/plugins/crunch/filter_latest_intersect.py +5 -4
- eodag/plugins/crunch/filter_latest_tpl_name.py +1 -1
- eodag/plugins/crunch/filter_overlap.py +5 -7
- eodag/plugins/crunch/filter_property.py +4 -3
- eodag/plugins/download/aws.py +39 -22
- eodag/plugins/download/base.py +11 -11
- eodag/plugins/download/creodias_s3.py +11 -2
- eodag/plugins/download/http.py +86 -52
- eodag/plugins/download/s3rest.py +20 -18
- eodag/plugins/manager.py +168 -23
- eodag/plugins/search/base.py +33 -14
- eodag/plugins/search/build_search_result.py +55 -51
- eodag/plugins/search/cop_marine.py +112 -29
- eodag/plugins/search/creodias_s3.py +20 -5
- eodag/plugins/search/csw.py +41 -1
- eodag/plugins/search/data_request_search.py +109 -9
- eodag/plugins/search/qssearch.py +532 -152
- eodag/plugins/search/static_stac_search.py +20 -21
- eodag/resources/ext_product_types.json +1 -1
- eodag/resources/product_types.yml +187 -56
- eodag/resources/providers.yml +1610 -1701
- eodag/resources/stac.yml +3 -163
- eodag/resources/user_conf_template.yml +112 -97
- eodag/rest/config.py +1 -2
- eodag/rest/constants.py +0 -1
- eodag/rest/core.py +61 -51
- eodag/rest/errors.py +181 -0
- eodag/rest/server.py +24 -325
- eodag/rest/stac.py +93 -544
- eodag/rest/types/eodag_search.py +13 -8
- eodag/rest/types/queryables.py +1 -2
- eodag/rest/types/stac_search.py +11 -2
- eodag/types/__init__.py +15 -3
- eodag/types/download_args.py +1 -1
- eodag/types/queryables.py +1 -2
- eodag/types/search_args.py +3 -3
- eodag/utils/__init__.py +77 -57
- eodag/utils/exceptions.py +23 -9
- eodag/utils/logging.py +37 -77
- eodag/utils/requests.py +1 -3
- eodag/utils/stac_reader.py +1 -1
- {eodag-3.0.0b3.dist-info → eodag-3.0.1.dist-info}/METADATA +11 -12
- eodag-3.0.1.dist-info/RECORD +109 -0
- {eodag-3.0.0b3.dist-info → eodag-3.0.1.dist-info}/WHEEL +1 -1
- {eodag-3.0.0b3.dist-info → eodag-3.0.1.dist-info}/entry_points.txt +1 -0
- eodag/resources/constraints/climate-dt.json +0 -13
- eodag/resources/constraints/extremes-dt.json +0 -8
- eodag-3.0.0b3.dist-info/RECORD +0 -110
- {eodag-3.0.0b3.dist-info → eodag-3.0.1.dist-info}/LICENSE +0 -0
- {eodag-3.0.0b3.dist-info → eodag-3.0.1.dist-info}/top_level.txt +0 -0
eodag/rest/types/eodag_search.py
CHANGED
|
@@ -115,13 +115,6 @@ class EODAGSearch(BaseModel):
|
|
|
115
115
|
|
|
116
116
|
_to_eodag_map: Dict[str, str]
|
|
117
117
|
|
|
118
|
-
@model_validator(mode="after")
|
|
119
|
-
def set_raise_errors(self) -> Self:
|
|
120
|
-
"""Set raise_errors to True if provider is set"""
|
|
121
|
-
if self.provider:
|
|
122
|
-
self.raise_errors = True
|
|
123
|
-
return self
|
|
124
|
-
|
|
125
118
|
@model_validator(mode="after")
|
|
126
119
|
def remove_timeFromAscendingNode(self) -> Self: # pylint: disable=invalid-name
|
|
127
120
|
"""TimeFromAscendingNode are just used for translation and not for search"""
|
|
@@ -367,9 +360,21 @@ class EODAGSearch(BaseModel):
|
|
|
367
360
|
return cls._to_eodag_map.get(value, value)
|
|
368
361
|
|
|
369
362
|
@classmethod
|
|
370
|
-
def to_stac(
|
|
363
|
+
def to_stac(
|
|
364
|
+
cls,
|
|
365
|
+
field_name: str,
|
|
366
|
+
stac_item_properties: Optional[List[str]] = None,
|
|
367
|
+
provider: Optional[str] = None,
|
|
368
|
+
) -> str:
|
|
371
369
|
"""Get the alias of a field in a Pydantic model"""
|
|
372
370
|
field = cls.model_fields.get(field_name)
|
|
373
371
|
if field is not None and field.alias is not None:
|
|
374
372
|
return field.alias
|
|
373
|
+
if (
|
|
374
|
+
provider
|
|
375
|
+
and ":" not in field_name
|
|
376
|
+
and stac_item_properties
|
|
377
|
+
and field_name not in stac_item_properties
|
|
378
|
+
):
|
|
379
|
+
return f"{provider}:{field_name}"
|
|
375
380
|
return field_name
|
eodag/rest/types/queryables.py
CHANGED
|
@@ -17,7 +17,7 @@
|
|
|
17
17
|
# limitations under the License.
|
|
18
18
|
from __future__ import annotations
|
|
19
19
|
|
|
20
|
-
from typing import TYPE_CHECKING, Any, ClassVar, Dict, List, Optional, Union
|
|
20
|
+
from typing import TYPE_CHECKING, Annotated, Any, ClassVar, Dict, List, Optional, Union
|
|
21
21
|
|
|
22
22
|
from pydantic import (
|
|
23
23
|
BaseModel,
|
|
@@ -32,7 +32,6 @@ from pydantic import (
|
|
|
32
32
|
from eodag.rest.types.eodag_search import EODAGSearch
|
|
33
33
|
from eodag.rest.utils.rfc3339 import str_to_interval
|
|
34
34
|
from eodag.types import python_field_definition_to_json
|
|
35
|
-
from eodag.utils import Annotated
|
|
36
35
|
|
|
37
36
|
if TYPE_CHECKING:
|
|
38
37
|
from pydantic.fields import FieldInfo
|
eodag/rest/types/stac_search.py
CHANGED
|
@@ -19,7 +19,17 @@
|
|
|
19
19
|
|
|
20
20
|
from __future__ import annotations
|
|
21
21
|
|
|
22
|
-
from typing import
|
|
22
|
+
from typing import (
|
|
23
|
+
TYPE_CHECKING,
|
|
24
|
+
Annotated,
|
|
25
|
+
Any,
|
|
26
|
+
Dict,
|
|
27
|
+
List,
|
|
28
|
+
Literal,
|
|
29
|
+
Optional,
|
|
30
|
+
Tuple,
|
|
31
|
+
Union,
|
|
32
|
+
)
|
|
23
33
|
|
|
24
34
|
import geojson
|
|
25
35
|
from pydantic import (
|
|
@@ -43,7 +53,6 @@ from shapely.geometry import (
|
|
|
43
53
|
shape,
|
|
44
54
|
)
|
|
45
55
|
from shapely.geometry.base import GEOMETRY_TYPES, BaseGeometry
|
|
46
|
-
from typing_extensions import Annotated
|
|
47
56
|
|
|
48
57
|
from eodag.rest.utils.rfc3339 import rfc3339_str_to_datetime, str_to_interval
|
|
49
58
|
from eodag.utils.exceptions import ValidationError
|
eodag/types/__init__.py
CHANGED
|
@@ -18,13 +18,25 @@
|
|
|
18
18
|
"""EODAG types"""
|
|
19
19
|
from __future__ import annotations
|
|
20
20
|
|
|
21
|
-
from typing import
|
|
21
|
+
from typing import (
|
|
22
|
+
Annotated,
|
|
23
|
+
Any,
|
|
24
|
+
Dict,
|
|
25
|
+
List,
|
|
26
|
+
Literal,
|
|
27
|
+
Optional,
|
|
28
|
+
Tuple,
|
|
29
|
+
TypedDict,
|
|
30
|
+
Union,
|
|
31
|
+
get_args,
|
|
32
|
+
get_origin,
|
|
33
|
+
)
|
|
22
34
|
|
|
23
35
|
from annotated_types import Gt, Lt
|
|
24
36
|
from pydantic import Field
|
|
25
37
|
from pydantic.fields import FieldInfo
|
|
26
38
|
|
|
27
|
-
from eodag.utils import
|
|
39
|
+
from eodag.utils import copy_deepcopy
|
|
28
40
|
from eodag.utils.exceptions import ValidationError
|
|
29
41
|
|
|
30
42
|
# Types mapping from JSON Schema and OpenAPI 3.1.0 specifications to Python
|
|
@@ -183,7 +195,7 @@ def python_field_definition_to_json(
|
|
|
183
195
|
"""Get json field definition from python `typing.Annotated`
|
|
184
196
|
|
|
185
197
|
>>> from pydantic import Field
|
|
186
|
-
>>> from
|
|
198
|
+
>>> from typing import Annotated
|
|
187
199
|
>>> python_field_definition_to_json(
|
|
188
200
|
... Annotated[
|
|
189
201
|
... Optional[str],
|
eodag/types/download_args.py
CHANGED
|
@@ -23,7 +23,7 @@ from typing import Dict, Optional, TypedDict
|
|
|
23
23
|
class DownloadConf(TypedDict, total=False):
|
|
24
24
|
"""Download configuration
|
|
25
25
|
|
|
26
|
-
:cvar
|
|
26
|
+
:cvar output_dir: where to store downloaded products, as an absolute file path
|
|
27
27
|
(Default: local temporary directory)
|
|
28
28
|
:cvar output_extension: downloaded file extension
|
|
29
29
|
:cvar extract: whether to extract the downloaded products, only applies to archived products
|
eodag/types/queryables.py
CHANGED
|
@@ -15,12 +15,11 @@
|
|
|
15
15
|
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
16
16
|
# See the License for the specific language governing permissions and
|
|
17
17
|
# limitations under the License.
|
|
18
|
-
from typing import Optional
|
|
18
|
+
from typing import Annotated, Optional
|
|
19
19
|
|
|
20
20
|
from annotated_types import Lt
|
|
21
21
|
from pydantic import BaseModel, Field
|
|
22
22
|
from pydantic.types import PositiveInt
|
|
23
|
-
from typing_extensions import Annotated
|
|
24
23
|
|
|
25
24
|
Percentage = Annotated[PositiveInt, Lt(100)]
|
|
26
25
|
|
eodag/types/search_args.py
CHANGED
|
@@ -17,7 +17,7 @@
|
|
|
17
17
|
# limitations under the License.
|
|
18
18
|
import re
|
|
19
19
|
from datetime import datetime
|
|
20
|
-
from typing import Any, Dict, List, Optional, Tuple, Union, cast
|
|
20
|
+
from typing import Annotated, Any, Dict, List, Optional, Tuple, Union, cast
|
|
21
21
|
|
|
22
22
|
from annotated_types import MinLen
|
|
23
23
|
from pydantic import BaseModel, ConfigDict, Field, conint, field_validator
|
|
@@ -27,7 +27,7 @@ from shapely.geometry import Polygon, shape
|
|
|
27
27
|
from shapely.geometry.base import GEOMETRY_TYPES, BaseGeometry
|
|
28
28
|
|
|
29
29
|
from eodag.types.bbox import BBox
|
|
30
|
-
from eodag.utils import DEFAULT_ITEMS_PER_PAGE, DEFAULT_PAGE
|
|
30
|
+
from eodag.utils import DEFAULT_ITEMS_PER_PAGE, DEFAULT_PAGE
|
|
31
31
|
from eodag.utils.exceptions import ValidationError
|
|
32
32
|
|
|
33
33
|
NumType = Union[float, int]
|
|
@@ -119,7 +119,7 @@ class SearchArgs(BaseModel):
|
|
|
119
119
|
)
|
|
120
120
|
sort_by_arg[i] = (sort_param, sort_order[:3])
|
|
121
121
|
# remove duplicates
|
|
122
|
-
pruned_sort_by_arg: SortByList = list(
|
|
122
|
+
pruned_sort_by_arg: SortByList = list(dict.fromkeys(sort_by_arg)) # type: ignore
|
|
123
123
|
for i, sort_by_tuple in enumerate(pruned_sort_by_arg):
|
|
124
124
|
for j, sort_by_tuple_tmp in enumerate(pruned_sort_by_arg):
|
|
125
125
|
# since duplicated tuples or dictionnaries have been removed, if two sorting parameters are equal,
|
eodag/utils/__init__.py
CHANGED
|
@@ -79,11 +79,6 @@ from urllib.parse import ( # noqa; noqa
|
|
|
79
79
|
)
|
|
80
80
|
from urllib.request import url2pathname
|
|
81
81
|
|
|
82
|
-
if sys.version_info >= (3, 9):
|
|
83
|
-
from typing import Annotated, get_args, get_origin # noqa
|
|
84
|
-
else:
|
|
85
|
-
from typing_extensions import Annotated, get_args, get_origin # type: ignore # noqa
|
|
86
|
-
|
|
87
82
|
if sys.version_info >= (3, 12):
|
|
88
83
|
from typing import Unpack # type: ignore # noqa
|
|
89
84
|
else:
|
|
@@ -99,7 +94,7 @@ from dateutil.tz import UTC
|
|
|
99
94
|
from jsonpath_ng import jsonpath
|
|
100
95
|
from jsonpath_ng.ext import parse
|
|
101
96
|
from jsonpath_ng.jsonpath import Child, Fields, Index, Root, Slice
|
|
102
|
-
from requests import HTTPError
|
|
97
|
+
from requests import HTTPError, Response
|
|
103
98
|
from shapely.geometry import Polygon, shape
|
|
104
99
|
from shapely.geometry.base import GEOMETRY_TYPES, BaseGeometry
|
|
105
100
|
from tqdm.auto import tqdm
|
|
@@ -110,7 +105,7 @@ from eodag.utils.exceptions import MisconfiguredError
|
|
|
110
105
|
if TYPE_CHECKING:
|
|
111
106
|
from jsonpath_ng import JSONPath
|
|
112
107
|
|
|
113
|
-
from eodag.api.product import EOProduct
|
|
108
|
+
from eodag.api.product._product import EOProduct
|
|
114
109
|
|
|
115
110
|
|
|
116
111
|
logger = py_logging.getLogger("eodag.utils")
|
|
@@ -125,6 +120,10 @@ USER_AGENT = {"User-Agent": f"eodag/{eodag_version}"}
|
|
|
125
120
|
HTTP_REQ_TIMEOUT = 5 # in seconds
|
|
126
121
|
DEFAULT_STREAM_REQUESTS_TIMEOUT = 60 # in seconds
|
|
127
122
|
|
|
123
|
+
REQ_RETRY_TOTAL = 3
|
|
124
|
+
REQ_RETRY_BACKOFF_FACTOR = 2
|
|
125
|
+
REQ_RETRY_STATUS_FORCELIST = [401, 429, 500, 502, 503, 504]
|
|
126
|
+
|
|
128
127
|
# default wait times in minutes
|
|
129
128
|
DEFAULT_DOWNLOAD_WAIT = 2 # in minutes
|
|
130
129
|
DEFAULT_DOWNLOAD_TIMEOUT = 20 # in minutes
|
|
@@ -237,9 +236,10 @@ class FloatRange(click.types.FloatParamType):
|
|
|
237
236
|
def slugify(value: Any, allow_unicode: bool = False) -> str:
|
|
238
237
|
"""Copied from Django Source code, only modifying last line (no need for safe
|
|
239
238
|
strings).
|
|
239
|
+
|
|
240
240
|
source: https://github.com/django/django/blob/master/django/utils/text.py
|
|
241
241
|
|
|
242
|
-
Convert to ASCII if
|
|
242
|
+
Convert to ASCII if ``allow_unicode`` is ``False``. Convert spaces to hyphens.
|
|
243
243
|
Remove characters that aren't alphanumerics, underscores, or hyphens.
|
|
244
244
|
Convert to lowercase. Also strip leading and trailing whitespace.
|
|
245
245
|
"""
|
|
@@ -298,7 +298,7 @@ def strip_accents(s: str) -> str:
|
|
|
298
298
|
|
|
299
299
|
def uri_to_path(uri: str) -> str:
|
|
300
300
|
"""
|
|
301
|
-
Convert a file URI (e.g.
|
|
301
|
+
Convert a file URI (e.g. ``file:///tmp``) to a local path (e.g. ``/tmp``)
|
|
302
302
|
"""
|
|
303
303
|
if not uri.startswith("file"):
|
|
304
304
|
raise ValueError("A file URI must be provided (e.g. 'file:///tmp'")
|
|
@@ -333,10 +333,10 @@ def mutate_dict_in_place(func: Callable[[Any], Any], mapping: Dict[Any, Any]) ->
|
|
|
333
333
|
|
|
334
334
|
|
|
335
335
|
def merge_mappings(mapping1: Dict[Any, Any], mapping2: Dict[Any, Any]) -> None:
|
|
336
|
-
"""Merge two mappings with string keys, values from
|
|
337
|
-
from
|
|
336
|
+
"""Merge two mappings with string keys, values from ``mapping2`` overriding values
|
|
337
|
+
from ``mapping1``.
|
|
338
338
|
|
|
339
|
-
Do its best to detect the key in
|
|
339
|
+
Do its best to detect the key in ``mapping1`` to override. For example:
|
|
340
340
|
|
|
341
341
|
>>> mapping2 = {"keya": "new"}
|
|
342
342
|
>>> mapping1 = {"keyA": "obsolete"}
|
|
@@ -344,12 +344,11 @@ def merge_mappings(mapping1: Dict[Any, Any], mapping2: Dict[Any, Any]) -> None:
|
|
|
344
344
|
>>> mapping1
|
|
345
345
|
{'keyA': 'new'}
|
|
346
346
|
|
|
347
|
-
If mapping2 has a key that cannot be detected in mapping1
|
|
348
|
-
to mapping1 as is.
|
|
347
|
+
If ``mapping2`` has a key that cannot be detected in ``mapping1``, this new key is
|
|
348
|
+
added to ``mapping1`` as is.
|
|
349
349
|
|
|
350
350
|
:param mapping1: The mapping containing values to be overridden
|
|
351
|
-
:param mapping2: The mapping containing values that will override the
|
|
352
|
-
first mapping
|
|
351
|
+
:param mapping2: The mapping containing values that will override the first mapping
|
|
353
352
|
"""
|
|
354
353
|
# A mapping between mapping1 keys as lowercase strings and original mapping1 keys
|
|
355
354
|
m1_keys_lowercase = {key.lower(): key for key in mapping1}
|
|
@@ -416,7 +415,7 @@ def get_timestamp(date_time: str) -> float:
|
|
|
416
415
|
If the datetime has no offset, it is assumed to be an UTC datetime.
|
|
417
416
|
|
|
418
417
|
:param date_time: The datetime string to return as timestamp
|
|
419
|
-
:returns: The timestamp corresponding to the date_time string in seconds
|
|
418
|
+
:returns: The timestamp corresponding to the ``date_time`` string in seconds
|
|
420
419
|
"""
|
|
421
420
|
dt = isoparse(date_time)
|
|
422
421
|
if not dt.tzinfo:
|
|
@@ -425,7 +424,7 @@ def get_timestamp(date_time: str) -> float:
|
|
|
425
424
|
|
|
426
425
|
|
|
427
426
|
def datetime_range(start: dt, end: dt) -> Iterator[dt]:
|
|
428
|
-
"""Generator function for all dates in-between start and end date."""
|
|
427
|
+
"""Generator function for all dates in-between ``start`` and ``end`` date."""
|
|
429
428
|
delta = end - start
|
|
430
429
|
for nday in range(delta.days + 1):
|
|
431
430
|
yield start + datetime.timedelta(days=nday)
|
|
@@ -445,15 +444,15 @@ class DownloadedCallback:
|
|
|
445
444
|
class ProgressCallback(tqdm):
|
|
446
445
|
"""A callable used to render progress to users for long running processes.
|
|
447
446
|
|
|
448
|
-
It inherits from
|
|
449
|
-
instantiation:
|
|
450
|
-
|
|
451
|
-
|
|
452
|
-
|
|
447
|
+
It inherits from :class:`tqdm.auto.tqdm`, and accepts the same arguments on
|
|
448
|
+
instantiation: ``iterable``, ``desc``, ``total``, ``leave``, ``file``, ``ncols``,
|
|
449
|
+
``mininterval``, ``maxinterval``, ``miniters``, ``ascii``, ``disable``, ``unit``,
|
|
450
|
+
``unit_scale``, ``dynamic_ncols``, ``smoothing``, ``bar_format``, ``initial``,
|
|
451
|
+
``position``, ``postfix``, ``unit_divisor``.
|
|
453
452
|
|
|
454
|
-
It can be globally disabled using
|
|
455
|
-
|
|
456
|
-
individually disabled using
|
|
453
|
+
It can be globally disabled using ``eodag.utils.logging.setup_logging(0)`` or
|
|
454
|
+
``eodag.utils.logging.setup_logging(level, no_progress_bar=True)``, and
|
|
455
|
+
individually disabled using ``disable=True``.
|
|
457
456
|
"""
|
|
458
457
|
|
|
459
458
|
def __init__(self, *args: Any, **kwargs: Any) -> None:
|
|
@@ -488,8 +487,8 @@ class ProgressCallback(tqdm):
|
|
|
488
487
|
"""Returns another progress callback using the same initial
|
|
489
488
|
keyword-arguments.
|
|
490
489
|
|
|
491
|
-
Optional
|
|
492
|
-
new
|
|
490
|
+
Optional ``args`` and ``kwargs`` parameters will be used to create a
|
|
491
|
+
new :class:`~eodag.utils.ProgressCallback` instance, overriding initial
|
|
493
492
|
`kwargs`.
|
|
494
493
|
"""
|
|
495
494
|
|
|
@@ -511,7 +510,7 @@ def get_progress_callback() -> tqdm:
|
|
|
511
510
|
|
|
512
511
|
|
|
513
512
|
def repeatfunc(func: Callable[..., Any], n: int, *args: Any) -> starmap:
|
|
514
|
-
"""Call
|
|
513
|
+
"""Call ``func`` ``n`` times with ``args``"""
|
|
515
514
|
return starmap(func, repeat(args, n))
|
|
516
515
|
|
|
517
516
|
|
|
@@ -526,12 +525,12 @@ def makedirs(dirpath: str) -> None:
|
|
|
526
525
|
|
|
527
526
|
|
|
528
527
|
def rename_subfolder(dirpath: str, name: str) -> None:
|
|
529
|
-
"""Rename first subfolder found in dirpath with given name
|
|
530
|
-
raise RuntimeError if no subfolder can be found
|
|
528
|
+
"""Rename first subfolder found in ``dirpath`` with given ``name``,
|
|
529
|
+
raise :class:`RuntimeError` if no subfolder can be found
|
|
531
530
|
|
|
532
531
|
:param dirpath: path to the directory containing the subfolder
|
|
533
532
|
:param name: new name of the subfolder
|
|
534
|
-
:raises: RuntimeError
|
|
533
|
+
:raises: :class:`RuntimeError`
|
|
535
534
|
|
|
536
535
|
Example:
|
|
537
536
|
|
|
@@ -545,16 +544,20 @@ def rename_subfolder(dirpath: str, name: str) -> None:
|
|
|
545
544
|
... rename_subfolder(tmpdir, "otherfolder")
|
|
546
545
|
... assert not os.path.isdir(somefolder) and os.path.isdir(otherfolder)
|
|
547
546
|
|
|
548
|
-
Before
|
|
547
|
+
Before::
|
|
548
|
+
|
|
549
549
|
$ tree <tmp-folder>
|
|
550
550
|
<tmp-folder>
|
|
551
551
|
└── somefolder
|
|
552
552
|
└── somefile
|
|
553
|
-
|
|
553
|
+
|
|
554
|
+
After::
|
|
555
|
+
|
|
554
556
|
$ tree <tmp-folder>
|
|
555
557
|
<tmp-folder>
|
|
556
558
|
└── otherfolder
|
|
557
559
|
└── somefile
|
|
560
|
+
|
|
558
561
|
"""
|
|
559
562
|
try:
|
|
560
563
|
subdir, *_ = (p for p in glob(os.path.join(dirpath, "*")) if os.path.isdir(p))
|
|
@@ -570,7 +573,7 @@ def rename_subfolder(dirpath: str, name: str) -> None:
|
|
|
570
573
|
def format_dict_items(
|
|
571
574
|
config_dict: Dict[str, Any], **format_variables: Any
|
|
572
575
|
) -> Dict[Any, Any]:
|
|
573
|
-
r"""
|
|
576
|
+
r"""Recursively apply :meth:`str.format` to ``**format_variables`` on ``config_dict`` values
|
|
574
577
|
|
|
575
578
|
>>> format_dict_items(
|
|
576
579
|
... {"foo": {"bar": "{a}"}, "baz": ["{b}?", "{b}!"]},
|
|
@@ -578,7 +581,7 @@ def format_dict_items(
|
|
|
578
581
|
... ) == {"foo": {"bar": "qux"}, "baz": ["quux?", "quux!"]}
|
|
579
582
|
True
|
|
580
583
|
|
|
581
|
-
:param config_dict:
|
|
584
|
+
:param config_dict: Dictionary having values that need to be parsed
|
|
582
585
|
:param format_variables: Variables used as args for parsing
|
|
583
586
|
:returns: Updated dict
|
|
584
587
|
"""
|
|
@@ -588,7 +591,7 @@ def format_dict_items(
|
|
|
588
591
|
def jsonpath_parse_dict_items(
|
|
589
592
|
jsonpath_dict: Dict[str, Any], values_dict: Dict[str, Any]
|
|
590
593
|
) -> Dict[Any, Any]:
|
|
591
|
-
"""
|
|
594
|
+
"""Recursively parse :class:`jsonpath_ng.JSONPath` elements in dict
|
|
592
595
|
|
|
593
596
|
>>> import jsonpath_ng.ext as jsonpath
|
|
594
597
|
>>> jsonpath_parse_dict_items(
|
|
@@ -597,7 +600,7 @@ def jsonpath_parse_dict_items(
|
|
|
597
600
|
... ) == {'foo': {'bar': 'baz'}, 'qux': ['quux', 'quux']}
|
|
598
601
|
True
|
|
599
602
|
|
|
600
|
-
:param jsonpath_dict:
|
|
603
|
+
:param jsonpath_dict: Dictionary having :class:`jsonpath_ng.JSONPath` values that need to be parsed
|
|
601
604
|
:param values_dict: Values dict used as args for parsing
|
|
602
605
|
:returns: Updated dict
|
|
603
606
|
"""
|
|
@@ -611,7 +614,7 @@ def update_nested_dict(
|
|
|
611
614
|
allow_empty_values: bool = False,
|
|
612
615
|
allow_extend_duplicates: bool = True,
|
|
613
616
|
) -> Dict[Any, Any]:
|
|
614
|
-
"""Update recursively old_dict items with new_dict ones
|
|
617
|
+
"""Update recursively ``old_dict`` items with ``new_dict`` ones
|
|
615
618
|
|
|
616
619
|
>>> update_nested_dict(
|
|
617
620
|
... {"a": {"a.a": 1, "a.b": 2}, "b": 3},
|
|
@@ -743,7 +746,7 @@ def dict_items_recursive_apply(
|
|
|
743
746
|
... ) == {'foo': {'bar': 'BAZ!'}, 'qux': ['A!', 'B!']}
|
|
744
747
|
True
|
|
745
748
|
|
|
746
|
-
:param config_dict: Input nested
|
|
749
|
+
:param config_dict: Input nested dictionary
|
|
747
750
|
:param apply_method: Method to be applied to dict elements
|
|
748
751
|
:param apply_method_parameters: Optional parameters passed to the method
|
|
749
752
|
:returns: Updated dict
|
|
@@ -836,7 +839,7 @@ def dict_items_recursive_sort(config_dict: Dict[Any, Any]) -> Dict[Any, Any]:
|
|
|
836
839
|
... ) == {"a": ["b", {0: 1, 1: 2, 2: 0}], "b": {"a": 0, "b": "c"}}
|
|
837
840
|
True
|
|
838
841
|
|
|
839
|
-
:param config_dict: Input nested
|
|
842
|
+
:param config_dict: Input nested dictionary
|
|
840
843
|
:returns: Updated dict
|
|
841
844
|
"""
|
|
842
845
|
result_dict: Dict[Any, Any] = deepcopy(config_dict)
|
|
@@ -873,7 +876,7 @@ def list_items_recursive_sort(config_list: List[Any]) -> List[Any]:
|
|
|
873
876
|
|
|
874
877
|
|
|
875
878
|
def string_to_jsonpath(*args: Any, force: bool = False) -> Union[str, JSONPath]:
|
|
876
|
-
"""Get
|
|
879
|
+
"""Get :class:`jsonpath_ng.JSONPath` for ``$.foo.bar`` like string
|
|
877
880
|
|
|
878
881
|
>>> string_to_jsonpath(None, "$.foo.bar")
|
|
879
882
|
Child(Child(Root(), Fields('foo')), Fields('bar'))
|
|
@@ -887,7 +890,7 @@ def string_to_jsonpath(*args: Any, force: bool = False) -> Union[str, JSONPath]:
|
|
|
887
890
|
Fields('foo')
|
|
888
891
|
|
|
889
892
|
:param args: Last arg as input string value, to be converted
|
|
890
|
-
:param force: force conversion even if input string is not detected as a
|
|
893
|
+
:param force: force conversion even if input string is not detected as a :class:`jsonpath_ng.JSONPath`
|
|
891
894
|
:returns: Parsed value
|
|
892
895
|
"""
|
|
893
896
|
path_str: str = args[-1]
|
|
@@ -950,7 +953,7 @@ def string_to_jsonpath(*args: Any, force: bool = False) -> Union[str, JSONPath]:
|
|
|
950
953
|
|
|
951
954
|
|
|
952
955
|
def format_string(key: str, str_to_format: Any, **format_variables: Any) -> Any:
|
|
953
|
-
"""Format "{foo}"
|
|
956
|
+
"""Format ``"{foo}"``-like string
|
|
954
957
|
|
|
955
958
|
>>> format_string(None, "foo {bar}, {baz} ?", **{"bar": "qux", "baz": "quux"})
|
|
956
959
|
'foo qux, quux ?'
|
|
@@ -988,7 +991,7 @@ def format_string(key: str, str_to_format: Any, **format_variables: Any) -> Any:
|
|
|
988
991
|
def parse_jsonpath(
|
|
989
992
|
key: str, jsonpath_obj: Union[str, jsonpath.Child], **values_dict: Dict[str, Any]
|
|
990
993
|
) -> Optional[str]:
|
|
991
|
-
"""Parse jsonpah in jsonpath_obj using values_dict
|
|
994
|
+
"""Parse jsonpah in ``jsonpath_obj`` using ``values_dict``
|
|
992
995
|
|
|
993
996
|
>>> import jsonpath_ng.ext as jsonpath
|
|
994
997
|
>>> parse_jsonpath(None, parse("$.foo.bar"), **{"foo": {"bar": "baz"}})
|
|
@@ -1030,10 +1033,10 @@ def nested_pairs2dict(pairs: Union[List[Any], Any]) -> Union[Any, Dict[Any, Any]
|
|
|
1030
1033
|
def get_geometry_from_various(
|
|
1031
1034
|
locations_config: List[Dict[str, Any]] = [], **query_args: Any
|
|
1032
1035
|
) -> BaseGeometry:
|
|
1033
|
-
"""Creates a shapely
|
|
1036
|
+
"""Creates a ``shapely.geometry`` using given query kwargs arguments
|
|
1034
1037
|
|
|
1035
1038
|
:param locations_config: (optional) EODAG locations configuration
|
|
1036
|
-
:param query_args: Query kwargs arguments from core.search
|
|
1039
|
+
:param query_args: Query kwargs arguments from :meth:`~eodag.api.core.EODataAccessGateway.search`
|
|
1037
1040
|
:returns: shapely Geometry found
|
|
1038
1041
|
:raises: :class:`ValueError`
|
|
1039
1042
|
"""
|
|
@@ -1129,7 +1132,7 @@ class MockResponse:
|
|
|
1129
1132
|
def raise_for_status(self) -> None:
|
|
1130
1133
|
"""raises an exception when the status is not ok"""
|
|
1131
1134
|
if self.status_code != 200:
|
|
1132
|
-
raise HTTPError()
|
|
1135
|
+
raise HTTPError(response=Response())
|
|
1133
1136
|
|
|
1134
1137
|
|
|
1135
1138
|
def md5sum(file_path: str) -> str:
|
|
@@ -1163,7 +1166,7 @@ def obj_md5sum(data: Any) -> str:
|
|
|
1163
1166
|
|
|
1164
1167
|
@functools.lru_cache()
|
|
1165
1168
|
def cached_parse(str_to_parse: str) -> JSONPath:
|
|
1166
|
-
"""Cached jsonpath_ng.ext.parse
|
|
1169
|
+
"""Cached :func:`jsonpath_ng.ext.parse`
|
|
1167
1170
|
|
|
1168
1171
|
>>> cached_parse.cache_clear()
|
|
1169
1172
|
>>> cached_parse("$.foo")
|
|
@@ -1179,8 +1182,8 @@ def cached_parse(str_to_parse: str) -> JSONPath:
|
|
|
1179
1182
|
>>> cached_parse.cache_info()
|
|
1180
1183
|
CacheInfo(hits=1, misses=2, maxsize=128, currsize=2)
|
|
1181
1184
|
|
|
1182
|
-
:param str_to_parse: string to parse as
|
|
1183
|
-
:returns: parsed
|
|
1185
|
+
:param str_to_parse: string to parse as :class:`jsonpath_ng.JSONPath`
|
|
1186
|
+
:returns: parsed :class:`jsonpath_ng.JSONPath`
|
|
1184
1187
|
"""
|
|
1185
1188
|
return parse(str_to_parse)
|
|
1186
1189
|
|
|
@@ -1194,7 +1197,7 @@ def _mutable_cached_yaml_load(config_path: str) -> Any:
|
|
|
1194
1197
|
|
|
1195
1198
|
|
|
1196
1199
|
def cached_yaml_load(config_path: str) -> Dict[str, Any]:
|
|
1197
|
-
"""Cached yaml.load
|
|
1200
|
+
"""Cached :func:`yaml.load`
|
|
1198
1201
|
|
|
1199
1202
|
:param config_path: path to the yaml configuration file
|
|
1200
1203
|
:returns: loaded yaml configuration
|
|
@@ -1209,7 +1212,7 @@ def _mutable_cached_yaml_load_all(config_path: str) -> List[Any]:
|
|
|
1209
1212
|
|
|
1210
1213
|
|
|
1211
1214
|
def cached_yaml_load_all(config_path: str) -> List[Any]:
|
|
1212
|
-
"""Cached yaml.load_all
|
|
1215
|
+
"""Cached :func:`yaml.load_all`
|
|
1213
1216
|
|
|
1214
1217
|
Load all configurations stored in the configuration file as separated yaml documents
|
|
1215
1218
|
|
|
@@ -1274,7 +1277,8 @@ def flatten_top_directories(
|
|
|
1274
1277
|
|
|
1275
1278
|
def deepcopy(sth: Any) -> Any:
|
|
1276
1279
|
"""Customized and faster deepcopy inspired by https://stackoverflow.com/a/45858907
|
|
1277
|
-
|
|
1280
|
+
|
|
1281
|
+
``_copy_list`` and ``_copy_dict`` dispatchers available for the moment
|
|
1278
1282
|
|
|
1279
1283
|
:param sth: Object to copy
|
|
1280
1284
|
:returns: Copied object
|
|
@@ -1339,7 +1343,7 @@ def cast_scalar_value(value: Any, new_type: Any) -> Any:
|
|
|
1339
1343
|
|
|
1340
1344
|
:param value: the scalar value to convert
|
|
1341
1345
|
:param new_type: the wanted type
|
|
1342
|
-
:returns: scalar value converted to new_type
|
|
1346
|
+
:returns: scalar ``value`` converted to ``new_type``
|
|
1343
1347
|
"""
|
|
1344
1348
|
if isinstance(value, str) and new_type is bool:
|
|
1345
1349
|
# Bool is a type with special meaning in Python, thus the special
|
|
@@ -1385,8 +1389,9 @@ def guess_extension(type: str) -> Optional[str]:
|
|
|
1385
1389
|
|
|
1386
1390
|
def get_ssl_context(ssl_verify: bool) -> ssl.SSLContext:
|
|
1387
1391
|
"""
|
|
1388
|
-
Returns an SSL context based on ssl_verify argument.
|
|
1389
|
-
|
|
1392
|
+
Returns an SSL context based on ``ssl_verify`` argument.
|
|
1393
|
+
|
|
1394
|
+
:param ssl_verify: :attr:`~eodag.config.PluginConfig.ssl_verify` parameter
|
|
1390
1395
|
:returns: An SSL context object.
|
|
1391
1396
|
"""
|
|
1392
1397
|
ctx = ssl.create_default_context()
|
|
@@ -1413,3 +1418,18 @@ def sort_dict(input_dict: Dict[str, Any]) -> Dict[str, Any]:
|
|
|
1413
1418
|
k: sort_dict(v) if isinstance(v, dict) else v
|
|
1414
1419
|
for k, v in sorted(input_dict.items())
|
|
1415
1420
|
}
|
|
1421
|
+
|
|
1422
|
+
|
|
1423
|
+
def dict_md5sum(input_dict: Dict[str, Any]) -> str:
|
|
1424
|
+
"""
|
|
1425
|
+
Hash nested dictionary
|
|
1426
|
+
|
|
1427
|
+
:param input_dict: input dict
|
|
1428
|
+
:returns: hash
|
|
1429
|
+
|
|
1430
|
+
>>> hd = dict_md5sum({"b": {"c": 1, "a": 2, "b": 3}, "a": 4})
|
|
1431
|
+
>>> hd
|
|
1432
|
+
'a195bcef1bb3b419e9e74b7cc5db8098'
|
|
1433
|
+
>>> assert(dict_md5sum({"a": 4, "b": {"b": 3, "c": 1, "a": 2}}) == hd)
|
|
1434
|
+
"""
|
|
1435
|
+
return obj_md5sum(sort_dict(input_dict))
|
eodag/utils/exceptions.py
CHANGED
|
@@ -17,10 +17,12 @@
|
|
|
17
17
|
# limitations under the License.
|
|
18
18
|
from __future__ import annotations
|
|
19
19
|
|
|
20
|
-
from typing import TYPE_CHECKING
|
|
20
|
+
from typing import TYPE_CHECKING, Annotated
|
|
21
21
|
|
|
22
22
|
if TYPE_CHECKING:
|
|
23
|
-
from typing import Optional, Set
|
|
23
|
+
from typing import Optional, Set
|
|
24
|
+
|
|
25
|
+
from typing_extensions import Doc
|
|
24
26
|
|
|
25
27
|
|
|
26
28
|
class EodagError(Exception):
|
|
@@ -84,14 +86,26 @@ class RequestError(EodagError):
|
|
|
84
86
|
"""An error indicating that a request has failed. Usually eodag functions
|
|
85
87
|
and methods should catch and skip this"""
|
|
86
88
|
|
|
87
|
-
|
|
88
|
-
|
|
89
|
+
status_code: Annotated[Optional[int], Doc("HTTP status code")] = None
|
|
90
|
+
|
|
91
|
+
@classmethod
|
|
92
|
+
def from_error(cls, error: Exception, msg: Optional[str] = None):
|
|
93
|
+
"""Generate a RequestError from an Exception"""
|
|
94
|
+
status_code = getattr(error, "code", None)
|
|
95
|
+
text = getattr(error, "msg", None)
|
|
96
|
+
|
|
97
|
+
response = getattr(error, "response", None)
|
|
98
|
+
# Explicitly test for None because response objects are considered false if they
|
|
99
|
+
# have a status code other than 200
|
|
100
|
+
if response is not None:
|
|
101
|
+
status_code = response.status_code
|
|
102
|
+
text = response.text
|
|
103
|
+
|
|
104
|
+
text = text or str(error)
|
|
89
105
|
|
|
90
|
-
|
|
91
|
-
|
|
92
|
-
|
|
93
|
-
repr += f"- {str(err_tuple)}"
|
|
94
|
-
return repr
|
|
106
|
+
e = cls(msg, text) if msg else cls(text)
|
|
107
|
+
e.status_code = status_code
|
|
108
|
+
return e
|
|
95
109
|
|
|
96
110
|
|
|
97
111
|
class NoMatchingProductType(EodagError):
|