eodag 2.12.1__py3-none-any.whl → 3.0.0b1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- eodag/api/core.py +434 -319
- eodag/api/product/__init__.py +5 -1
- eodag/api/product/_assets.py +7 -2
- eodag/api/product/_product.py +46 -68
- eodag/api/product/metadata_mapping.py +181 -66
- eodag/api/search_result.py +21 -1
- eodag/cli.py +20 -6
- eodag/config.py +95 -6
- eodag/plugins/apis/base.py +8 -165
- eodag/plugins/apis/ecmwf.py +36 -24
- eodag/plugins/apis/usgs.py +40 -24
- eodag/plugins/authentication/aws_auth.py +2 -2
- eodag/plugins/authentication/header.py +31 -6
- eodag/plugins/authentication/keycloak.py +13 -84
- eodag/plugins/authentication/oauth.py +3 -3
- eodag/plugins/authentication/openid_connect.py +256 -46
- eodag/plugins/authentication/qsauth.py +3 -0
- eodag/plugins/authentication/sas_auth.py +8 -1
- eodag/plugins/authentication/token.py +92 -46
- eodag/plugins/authentication/token_exchange.py +120 -0
- eodag/plugins/download/aws.py +86 -91
- eodag/plugins/download/base.py +72 -40
- eodag/plugins/download/http.py +607 -264
- eodag/plugins/download/s3rest.py +28 -15
- eodag/plugins/manager.py +73 -57
- eodag/plugins/search/__init__.py +36 -0
- eodag/plugins/search/base.py +225 -18
- eodag/plugins/search/build_search_result.py +389 -32
- eodag/plugins/search/cop_marine.py +378 -0
- eodag/plugins/search/creodias_s3.py +15 -14
- eodag/plugins/search/csw.py +5 -7
- eodag/plugins/search/data_request_search.py +44 -20
- eodag/plugins/search/qssearch.py +508 -203
- eodag/plugins/search/static_stac_search.py +99 -36
- eodag/resources/constraints/climate-dt.json +13 -0
- eodag/resources/constraints/extremes-dt.json +8 -0
- eodag/resources/ext_product_types.json +1 -1
- eodag/resources/product_types.yml +1897 -34
- eodag/resources/providers.yml +3539 -3277
- eodag/resources/stac.yml +48 -54
- eodag/resources/stac_api.yml +71 -25
- eodag/resources/stac_provider.yml +5 -0
- eodag/resources/user_conf_template.yml +51 -3
- eodag/rest/__init__.py +6 -0
- eodag/rest/cache.py +70 -0
- eodag/rest/config.py +68 -0
- eodag/rest/constants.py +27 -0
- eodag/rest/core.py +757 -0
- eodag/rest/server.py +397 -258
- eodag/rest/stac.py +438 -307
- eodag/rest/types/collections_search.py +44 -0
- eodag/rest/types/eodag_search.py +232 -43
- eodag/rest/types/{stac_queryables.py → queryables.py} +81 -43
- eodag/rest/types/stac_search.py +277 -0
- eodag/rest/utils/__init__.py +216 -0
- eodag/rest/utils/cql_evaluate.py +119 -0
- eodag/rest/utils/rfc3339.py +65 -0
- eodag/types/__init__.py +99 -9
- eodag/types/bbox.py +15 -14
- eodag/types/download_args.py +31 -0
- eodag/types/search_args.py +58 -7
- eodag/types/whoosh.py +81 -0
- eodag/utils/__init__.py +72 -9
- eodag/utils/constraints.py +37 -37
- eodag/utils/exceptions.py +23 -17
- eodag/utils/requests.py +138 -0
- eodag/utils/rest.py +104 -0
- eodag/utils/stac_reader.py +100 -16
- {eodag-2.12.1.dist-info → eodag-3.0.0b1.dist-info}/METADATA +64 -44
- eodag-3.0.0b1.dist-info/RECORD +109 -0
- {eodag-2.12.1.dist-info → eodag-3.0.0b1.dist-info}/WHEEL +1 -1
- {eodag-2.12.1.dist-info → eodag-3.0.0b1.dist-info}/entry_points.txt +6 -5
- eodag/plugins/apis/cds.py +0 -540
- eodag/rest/utils.py +0 -1133
- eodag-2.12.1.dist-info/RECORD +0 -94
- {eodag-2.12.1.dist-info → eodag-3.0.0b1.dist-info}/LICENSE +0 -0
- {eodag-2.12.1.dist-info → eodag-3.0.0b1.dist-info}/top_level.txt +0 -0
eodag/types/__init__.py
CHANGED
|
@@ -18,8 +18,9 @@
|
|
|
18
18
|
"""EODAG types"""
|
|
19
19
|
from __future__ import annotations
|
|
20
20
|
|
|
21
|
-
from typing import Any, Dict, List, Literal, Optional, Union
|
|
21
|
+
from typing import Any, Dict, List, Literal, Optional, Tuple, TypedDict, Union
|
|
22
22
|
|
|
23
|
+
from annotated_types import Gt, Lt
|
|
23
24
|
from pydantic import Field
|
|
24
25
|
from pydantic.fields import FieldInfo
|
|
25
26
|
|
|
@@ -56,13 +57,51 @@ def json_type_to_python(json_type: Union[str, List[str]]) -> type:
|
|
|
56
57
|
return type(None)
|
|
57
58
|
|
|
58
59
|
|
|
59
|
-
def
|
|
60
|
+
def _get_min_or_max(type_info: Union[Lt, Gt, Any]) -> Tuple[str, Any]:
|
|
61
|
+
"""
|
|
62
|
+
checks if the value from an Annotated object is a minimum or maximum
|
|
63
|
+
:param type_info: info from Annotated
|
|
64
|
+
:return: "min" or "max"
|
|
65
|
+
"""
|
|
66
|
+
if isinstance(type_info, Gt):
|
|
67
|
+
return "min", type_info.gt
|
|
68
|
+
if isinstance(type_info, Lt):
|
|
69
|
+
return "max", type_info.lt
|
|
70
|
+
return "", None
|
|
71
|
+
|
|
72
|
+
|
|
73
|
+
def _get_type_info_from_annotated(
|
|
74
|
+
annotated_type: Annotated[type, Any]
|
|
75
|
+
) -> Dict[str, Any]:
|
|
76
|
+
"""
|
|
77
|
+
retrieves type information from an annotated object
|
|
78
|
+
:param annotated_type: annotated object
|
|
79
|
+
:return: dict containing type and min/max if available
|
|
80
|
+
"""
|
|
81
|
+
type_args = get_args(annotated_type)
|
|
82
|
+
type_data = {
|
|
83
|
+
"type": list(JSON_TYPES_MAPPING.keys())[
|
|
84
|
+
list(JSON_TYPES_MAPPING.values()).index(type_args[0])
|
|
85
|
+
]
|
|
86
|
+
}
|
|
87
|
+
if len(type_args) >= 2:
|
|
88
|
+
min_or_max, value = _get_min_or_max(type_args[1])
|
|
89
|
+
type_data[min_or_max] = value
|
|
90
|
+
if len(type_args) > 2:
|
|
91
|
+
min_or_max, value = _get_min_or_max(type_args[2])
|
|
92
|
+
type_data[min_or_max] = value
|
|
93
|
+
return type_data
|
|
94
|
+
|
|
95
|
+
|
|
96
|
+
def python_type_to_json(
|
|
97
|
+
python_type: type,
|
|
98
|
+
) -> Optional[Union[str, List[Dict[str, Any]]]]:
|
|
60
99
|
"""Get json type from python https://spec.openapis.org/oas/v3.1.0#data-types
|
|
61
100
|
|
|
62
101
|
>>> python_type_to_json(int)
|
|
63
102
|
'integer'
|
|
64
103
|
>>> python_type_to_json(Union[float, str])
|
|
65
|
-
['number', 'string']
|
|
104
|
+
[{'type': 'number'}, {'type': 'string'}]
|
|
66
105
|
|
|
67
106
|
:param python_type: the python type
|
|
68
107
|
:returns: the json type
|
|
@@ -70,18 +109,25 @@ def python_type_to_json(python_type: type) -> Optional[Union[str, List[str]]]:
|
|
|
70
109
|
if get_origin(python_type) is Union:
|
|
71
110
|
json_type = list()
|
|
72
111
|
for single_python_type in get_args(python_type):
|
|
112
|
+
type_data = {}
|
|
73
113
|
if single_python_type in JSON_TYPES_MAPPING.values():
|
|
74
114
|
# JSON_TYPES_MAPPING key from given value
|
|
75
115
|
single_json_type = list(JSON_TYPES_MAPPING.keys())[
|
|
76
116
|
list(JSON_TYPES_MAPPING.values()).index(single_python_type)
|
|
77
117
|
]
|
|
78
|
-
|
|
118
|
+
type_data["type"] = single_json_type
|
|
119
|
+
json_type.append(type_data)
|
|
120
|
+
elif get_origin(single_python_type) == Annotated:
|
|
121
|
+
type_data = _get_type_info_from_annotated(single_python_type)
|
|
122
|
+
json_type.append(type_data)
|
|
79
123
|
return json_type
|
|
80
124
|
elif python_type in JSON_TYPES_MAPPING.values():
|
|
81
125
|
# JSON_TYPES_MAPPING key from given value
|
|
82
126
|
return list(JSON_TYPES_MAPPING.keys())[
|
|
83
127
|
list(JSON_TYPES_MAPPING.values()).index(python_type)
|
|
84
128
|
]
|
|
129
|
+
elif get_origin(python_type) == Annotated:
|
|
130
|
+
return [_get_type_info_from_annotated(python_type)]
|
|
85
131
|
else:
|
|
86
132
|
return None
|
|
87
133
|
|
|
@@ -99,7 +145,9 @@ def json_field_definition_to_python(
|
|
|
99
145
|
... 'title': 'Foo parameter'
|
|
100
146
|
... }
|
|
101
147
|
... )
|
|
102
|
-
>>> str(result).replace('_extensions', '') # python3.8 compatibility
|
|
148
|
+
>>> res_repr = str(result).replace('_extensions', '') # python3.8 compatibility
|
|
149
|
+
>>> res_repr = res_repr.replace(', default=None', '') # pydantic >= 2.7.0 compatibility
|
|
150
|
+
>>> res_repr
|
|
103
151
|
"typing.Annotated[bool, FieldInfo(annotation=NoneType, required=False, title='Foo parameter')]"
|
|
104
152
|
|
|
105
153
|
:param json_field_definition: the json field definition
|
|
@@ -159,13 +207,39 @@ def python_field_definition_to_json(
|
|
|
159
207
|
# enum & type
|
|
160
208
|
if get_origin(python_field_args[0]) is Literal:
|
|
161
209
|
enum_args = get_args(python_field_args[0])
|
|
162
|
-
|
|
210
|
+
type_data = python_type_to_json(type(enum_args[0]))
|
|
211
|
+
if isinstance(type_data, str):
|
|
212
|
+
json_field_definition["type"] = type_data
|
|
213
|
+
else:
|
|
214
|
+
json_field_definition["type"] = [row["type"] for row in type_data]
|
|
215
|
+
json_field_definition["min"] = [
|
|
216
|
+
row["min"] if "min" in row else None for row in type_data
|
|
217
|
+
]
|
|
218
|
+
json_field_definition["max"] = [
|
|
219
|
+
row["max"] if "max" in row else None for row in type_data
|
|
220
|
+
]
|
|
163
221
|
json_field_definition["enum"] = list(enum_args)
|
|
164
222
|
# type
|
|
165
223
|
else:
|
|
166
224
|
field_type = python_type_to_json(python_field_args[0])
|
|
167
|
-
if field_type
|
|
168
|
-
json_field_definition["type"] =
|
|
225
|
+
if isinstance(field_type, str):
|
|
226
|
+
json_field_definition["type"] = field_type
|
|
227
|
+
else:
|
|
228
|
+
json_field_definition["type"] = [row["type"] for row in field_type]
|
|
229
|
+
json_field_definition["min"] = [
|
|
230
|
+
row["min"] if "min" in row else None for row in field_type
|
|
231
|
+
]
|
|
232
|
+
json_field_definition["max"] = [
|
|
233
|
+
row["max"] if "max" in row else None for row in field_type
|
|
234
|
+
]
|
|
235
|
+
if "min" in json_field_definition and json_field_definition["min"].count(
|
|
236
|
+
None
|
|
237
|
+
) == len(json_field_definition["min"]):
|
|
238
|
+
json_field_definition.pop("min")
|
|
239
|
+
if "max" in json_field_definition and json_field_definition["max"].count(
|
|
240
|
+
None
|
|
241
|
+
) == len(json_field_definition["max"]):
|
|
242
|
+
json_field_definition.pop("max")
|
|
169
243
|
|
|
170
244
|
if len(python_field_args) < 2:
|
|
171
245
|
return json_field_definition
|
|
@@ -204,7 +278,9 @@ def model_fields_to_annotated(
|
|
|
204
278
|
>>> from pydantic import create_model
|
|
205
279
|
>>> some_model = create_model("some_model", foo=(str, None))
|
|
206
280
|
>>> fields_definitions = model_fields_to_annotated(some_model.model_fields)
|
|
207
|
-
>>> str(fields_definitions).replace('_extensions', '') # python3.8 compatibility
|
|
281
|
+
>>> fd_repr = str(fields_definitions).replace('_extensions', '') # python3.8 compatibility
|
|
282
|
+
>>> fd_repr = fd_repr.replace(', default=None', '') # pydantic >= 2.7.0 compatibility
|
|
283
|
+
>>> fd_repr
|
|
208
284
|
"{'foo': typing.Annotated[str, FieldInfo(annotation=NoneType, required=False)]}"
|
|
209
285
|
|
|
210
286
|
:param model_fields: BaseModel.model_fields to convert
|
|
@@ -217,3 +293,17 @@ def model_fields_to_annotated(
|
|
|
217
293
|
new_field_info.annotation = None
|
|
218
294
|
annotated_model_fields[param] = Annotated[field_type, new_field_info]
|
|
219
295
|
return annotated_model_fields
|
|
296
|
+
|
|
297
|
+
|
|
298
|
+
class ProviderSortables(TypedDict):
|
|
299
|
+
"""A class representing sortable parameter(s) of a provider and the allowed
|
|
300
|
+
maximum number of used sortable(s) in a search request with the provider
|
|
301
|
+
|
|
302
|
+
:param sortables: The list of sortable parameter(s) of a provider
|
|
303
|
+
:type sortables: list[str]
|
|
304
|
+
:param max_sort_params: (optional) The allowed maximum number of sortable(s) in a search request with the provider
|
|
305
|
+
:type max_sort_params: int
|
|
306
|
+
"""
|
|
307
|
+
|
|
308
|
+
sortables: List[str]
|
|
309
|
+
max_sort_params: Annotated[Optional[int], Gt(0)]
|
eodag/types/bbox.py
CHANGED
|
@@ -15,9 +15,9 @@
|
|
|
15
15
|
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
16
16
|
# See the License for the specific language governing permissions and
|
|
17
17
|
# limitations under the License.
|
|
18
|
-
from typing import
|
|
18
|
+
from typing import Dict, List, Tuple, Union
|
|
19
19
|
|
|
20
|
-
from pydantic import BaseModel,
|
|
20
|
+
from pydantic import BaseModel, ValidationInfo, field_validator
|
|
21
21
|
from shapely.geometry.polygon import Polygon
|
|
22
22
|
|
|
23
23
|
NumType = Union[float, int]
|
|
@@ -36,7 +36,7 @@ class BBox(BaseModel):
|
|
|
36
36
|
lonmax: NumType
|
|
37
37
|
latmax: NumType
|
|
38
38
|
|
|
39
|
-
def __init__(__pydantic_self__, bboxArgs: BBoxArgs) -> None: # type: ignore
|
|
39
|
+
def __init__(__pydantic_self__, bboxArgs: BBoxArgs) -> None: # type: ignore # pylint: disable=no-self-argument
|
|
40
40
|
"""
|
|
41
41
|
Constructs all the necessary attributes for the BBox object.
|
|
42
42
|
|
|
@@ -55,11 +55,12 @@ class BBox(BaseModel):
|
|
|
55
55
|
values = bboxArgs
|
|
56
56
|
else:
|
|
57
57
|
raise ValueError(
|
|
58
|
-
"Expected a dictionary,
|
|
58
|
+
"Expected a dictionary,",
|
|
59
|
+
" list or tuple with 4 values for lonmin, latmin, lonmax, latmax",
|
|
59
60
|
)
|
|
60
61
|
super().__init__(**values)
|
|
61
62
|
|
|
62
|
-
@
|
|
63
|
+
@field_validator("lonmin", "lonmax")
|
|
63
64
|
@classmethod
|
|
64
65
|
def validate_longitude(cls, v: NumType) -> NumType:
|
|
65
66
|
"""
|
|
@@ -72,7 +73,7 @@ class BBox(BaseModel):
|
|
|
72
73
|
raise ValueError("Longitude values must be between -180 and 180")
|
|
73
74
|
return v
|
|
74
75
|
|
|
75
|
-
@
|
|
76
|
+
@field_validator("latmin", "latmax")
|
|
76
77
|
@classmethod
|
|
77
78
|
def validate_latitude(cls, v: NumType) -> NumType:
|
|
78
79
|
"""
|
|
@@ -85,31 +86,31 @@ class BBox(BaseModel):
|
|
|
85
86
|
raise ValueError("Latitude values must be between -90 and 90")
|
|
86
87
|
return v
|
|
87
88
|
|
|
88
|
-
@
|
|
89
|
+
@field_validator("lonmax")
|
|
89
90
|
@classmethod
|
|
90
|
-
def validate_lonmax(cls, v: NumType,
|
|
91
|
+
def validate_lonmax(cls, v: NumType, info: ValidationInfo) -> NumType:
|
|
91
92
|
"""
|
|
92
93
|
Validates that lonmax is greater than lonmin.
|
|
93
94
|
|
|
94
95
|
:param v: The lonmax value to be validated.
|
|
95
|
-
:param
|
|
96
|
+
:param info: Additional validation informations.
|
|
96
97
|
:return: The validated lonmax value.
|
|
97
98
|
"""
|
|
98
|
-
if "lonmin" in
|
|
99
|
+
if "lonmin" in info.data and v < info.data["lonmin"]:
|
|
99
100
|
raise ValueError("lonmax must be greater than lonmin")
|
|
100
101
|
return v
|
|
101
102
|
|
|
102
|
-
@
|
|
103
|
+
@field_validator("latmax")
|
|
103
104
|
@classmethod
|
|
104
|
-
def validate_latmax(cls, v: NumType,
|
|
105
|
+
def validate_latmax(cls, v: NumType, info: ValidationInfo) -> NumType:
|
|
105
106
|
"""
|
|
106
107
|
Validates that latmax is greater than latmin.
|
|
107
108
|
|
|
108
109
|
:param v: The latmax value to be validated.
|
|
109
|
-
:param
|
|
110
|
+
:param info: Additional validation informations.
|
|
110
111
|
:return: The validated latmax value.
|
|
111
112
|
"""
|
|
112
|
-
if "latmin" in
|
|
113
|
+
if "latmin" in info.data and v < info.data["latmin"]:
|
|
113
114
|
raise ValueError("latmax must be greater than latmin")
|
|
114
115
|
return v
|
|
115
116
|
|
|
@@ -0,0 +1,31 @@
|
|
|
1
|
+
# -*- coding: utf-8 -*-
|
|
2
|
+
# Copyright 2024, CS GROUP - France, https://www.csgroup.eu/
|
|
3
|
+
#
|
|
4
|
+
# This file is part of EODAG project
|
|
5
|
+
# https://www.github.com/CS-SI/EODAG
|
|
6
|
+
#
|
|
7
|
+
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
8
|
+
# you may not use this file except in compliance with the License.
|
|
9
|
+
# You may obtain a copy of the License at
|
|
10
|
+
#
|
|
11
|
+
# http://www.apache.org/licenses/LICENSE-2.0
|
|
12
|
+
#
|
|
13
|
+
# Unless required by applicable law or agreed to in writing, software
|
|
14
|
+
# distributed under the License is distributed on an "AS IS" BASIS,
|
|
15
|
+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
16
|
+
# See the License for the specific language governing permissions and
|
|
17
|
+
# limitations under the License.
|
|
18
|
+
from __future__ import annotations
|
|
19
|
+
|
|
20
|
+
from typing import Dict, TypedDict
|
|
21
|
+
|
|
22
|
+
|
|
23
|
+
class DownloadConf(TypedDict, total=False):
|
|
24
|
+
"""Download configuration"""
|
|
25
|
+
|
|
26
|
+
outputs_prefix: str
|
|
27
|
+
outputs_extension: str
|
|
28
|
+
extract: bool
|
|
29
|
+
dl_url_params: Dict[str, str]
|
|
30
|
+
delete_archive: bool
|
|
31
|
+
asset: str
|
eodag/types/search_args.py
CHANGED
|
@@ -15,9 +15,11 @@
|
|
|
15
15
|
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
16
16
|
# See the License for the specific language governing permissions and
|
|
17
17
|
# limitations under the License.
|
|
18
|
+
import re
|
|
18
19
|
from datetime import datetime
|
|
19
|
-
from typing import Dict, List, Optional, Tuple, Union, cast
|
|
20
|
+
from typing import Any, Dict, List, Optional, Tuple, Union, cast
|
|
20
21
|
|
|
22
|
+
from annotated_types import MinLen
|
|
21
23
|
from pydantic import BaseModel, ConfigDict, Field, conint, field_validator
|
|
22
24
|
from shapely import wkt
|
|
23
25
|
from shapely.errors import GEOSException
|
|
@@ -25,12 +27,14 @@ from shapely.geometry import Polygon, shape
|
|
|
25
27
|
from shapely.geometry.base import GEOMETRY_TYPES, BaseGeometry
|
|
26
28
|
|
|
27
29
|
from eodag.types.bbox import BBox
|
|
28
|
-
from eodag.utils import DEFAULT_ITEMS_PER_PAGE, DEFAULT_PAGE
|
|
30
|
+
from eodag.utils import DEFAULT_ITEMS_PER_PAGE, DEFAULT_PAGE, Annotated
|
|
31
|
+
from eodag.utils.exceptions import ValidationError
|
|
29
32
|
|
|
30
33
|
NumType = Union[float, int]
|
|
31
34
|
GeomArgs = Union[List[NumType], Tuple[NumType], Dict[str, NumType], str, BaseGeometry]
|
|
32
35
|
|
|
33
36
|
PositiveInt = conint(gt=0)
|
|
37
|
+
SortByList = Annotated[List[Tuple[str, str]], MinLen(1)]
|
|
34
38
|
|
|
35
39
|
|
|
36
40
|
class SearchArgs(BaseModel):
|
|
@@ -47,6 +51,7 @@ class SearchArgs(BaseModel):
|
|
|
47
51
|
locations: Optional[Dict[str, str]] = Field(None)
|
|
48
52
|
page: Optional[int] = Field(DEFAULT_PAGE, gt=0) # type: ignore
|
|
49
53
|
items_per_page: Optional[PositiveInt] = Field(DEFAULT_ITEMS_PER_PAGE) # type: ignore
|
|
54
|
+
sortBy: Optional[SortByList] = Field(None) # type: ignore
|
|
50
55
|
|
|
51
56
|
@field_validator("start", "end", mode="before")
|
|
52
57
|
@classmethod
|
|
@@ -60,24 +65,70 @@ class SearchArgs(BaseModel):
|
|
|
60
65
|
|
|
61
66
|
@field_validator("geom", mode="before")
|
|
62
67
|
@classmethod
|
|
63
|
-
def check_geom(cls, v:
|
|
68
|
+
def check_geom(cls, v: Any) -> BaseGeometry:
|
|
64
69
|
"""Validate geom"""
|
|
65
70
|
# GeoJSON geometry
|
|
66
|
-
if isinstance(v, dict) and v.get("type") in GEOMETRY_TYPES:
|
|
71
|
+
if isinstance(v, dict) and v.get("type") in GEOMETRY_TYPES: # type: ignore
|
|
67
72
|
return cast(BaseGeometry, shape(v))
|
|
68
73
|
|
|
69
74
|
# Bounding Box
|
|
70
75
|
if isinstance(v, (list, tuple, dict)):
|
|
71
|
-
return BBox(v).to_polygon()
|
|
76
|
+
return BBox(v).to_polygon() # type: ignore
|
|
72
77
|
|
|
73
78
|
if isinstance(v, str):
|
|
74
79
|
# WKT geometry
|
|
75
80
|
try:
|
|
76
|
-
return cast(Polygon, wkt.loads(v))
|
|
77
|
-
except GEOSException as e:
|
|
81
|
+
return cast(Polygon, wkt.loads(v)) # type: ignore
|
|
82
|
+
except GEOSException as e: # type: ignore
|
|
78
83
|
raise ValueError(f"Invalid geometry WKT string: {v}") from e
|
|
79
84
|
|
|
80
85
|
if isinstance(v, BaseGeometry):
|
|
81
86
|
return v
|
|
82
87
|
|
|
83
88
|
raise TypeError(f"Invalid geometry type: {type(v)}")
|
|
89
|
+
|
|
90
|
+
@field_validator("sortBy", mode="before")
|
|
91
|
+
@classmethod
|
|
92
|
+
def check_sort_by_arg(
|
|
93
|
+
cls, sort_by_arg: Optional[SortByList] # type: ignore
|
|
94
|
+
) -> Optional[SortByList]: # type: ignore
|
|
95
|
+
"""Check if the sortBy argument is correct
|
|
96
|
+
|
|
97
|
+
:param sort_by_arg: The sortBy argument
|
|
98
|
+
:type sort_by_arg: str
|
|
99
|
+
:returns: The sortBy argument with sorting order parsed (whitespace(s) are
|
|
100
|
+
removed and only the 3 first letters in uppercase are kept)
|
|
101
|
+
:rtype: str
|
|
102
|
+
"""
|
|
103
|
+
if sort_by_arg is None:
|
|
104
|
+
return None
|
|
105
|
+
|
|
106
|
+
assert isinstance(
|
|
107
|
+
sort_by_arg, list
|
|
108
|
+
), f"Sort argument must be a list of tuple(s), got a '{type(sort_by_arg)}' instead"
|
|
109
|
+
sort_order_pattern = r"^(ASC|DES)[a-zA-Z]*$"
|
|
110
|
+
for i, sort_by_tuple in enumerate(sort_by_arg):
|
|
111
|
+
assert isinstance(
|
|
112
|
+
sort_by_tuple, tuple
|
|
113
|
+
), f"Sort argument must be a list of tuple(s), got a list of '{type(sort_by_tuple)}' instead"
|
|
114
|
+
# get sorting elements by removing leading and trailing whitespace(s) if exist
|
|
115
|
+
sort_param = sort_by_tuple[0].strip()
|
|
116
|
+
sort_order = sort_by_tuple[1].strip().upper()
|
|
117
|
+
assert re.match(sort_order_pattern, sort_order) is not None, (
|
|
118
|
+
"Sorting order must be set to 'ASC' (ASCENDING) or 'DESC' (DESCENDING), "
|
|
119
|
+
f"got '{sort_order}' with '{sort_param}' instead"
|
|
120
|
+
)
|
|
121
|
+
sort_by_arg[i] = (sort_param, sort_order[:3])
|
|
122
|
+
# remove duplicates
|
|
123
|
+
pruned_sort_by_arg: SortByList = list(set(sort_by_arg)) # type: ignore
|
|
124
|
+
for i, sort_by_tuple in enumerate(pruned_sort_by_arg):
|
|
125
|
+
for j, sort_by_tuple_tmp in enumerate(pruned_sort_by_arg):
|
|
126
|
+
# since duplicated tuples or dictionnaries have been removed, if two sorting parameters are equal,
|
|
127
|
+
# then their sorting order is different and there is a contradiction that would raise an error
|
|
128
|
+
if i != j and sort_by_tuple[0] == sort_by_tuple_tmp[0]:
|
|
129
|
+
raise ValidationError(
|
|
130
|
+
f"'{sort_by_tuple[0]}' parameter is called several times to sort results with different "
|
|
131
|
+
"sorting orders. Please set it to only one ('ASC' (ASCENDING) or 'DESC' (DESCENDING))",
|
|
132
|
+
set([sort_by_tuple[0]]),
|
|
133
|
+
)
|
|
134
|
+
return pruned_sort_by_arg
|
eodag/types/whoosh.py
ADDED
|
@@ -0,0 +1,81 @@
|
|
|
1
|
+
# -*- coding: utf-8 -*-
|
|
2
|
+
# Copyright 2024, CS Systemes d'Information, https://www.csgroup.eu/
|
|
3
|
+
#
|
|
4
|
+
# This file is part of EODAG project
|
|
5
|
+
# https://www.github.com/CS-SI/EODAG
|
|
6
|
+
#
|
|
7
|
+
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
8
|
+
# you may not use this file except in compliance with the License.
|
|
9
|
+
# You may obtain a copy of the License at
|
|
10
|
+
#
|
|
11
|
+
# http://www.apache.org/licenses/LICENSE-2.0
|
|
12
|
+
#
|
|
13
|
+
# Unless required by applicable law or agreed to in writing, software
|
|
14
|
+
# distributed under the License is distributed on an "AS IS" BASIS,
|
|
15
|
+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
16
|
+
# See the License for the specific language governing permissions and
|
|
17
|
+
# limitations under the License.
|
|
18
|
+
from typing import List
|
|
19
|
+
|
|
20
|
+
from whoosh.fields import Schema
|
|
21
|
+
from whoosh.matching import NullMatcher
|
|
22
|
+
from whoosh.qparser import OrGroup, QueryParser, plugins
|
|
23
|
+
from whoosh.query.positional import Phrase
|
|
24
|
+
from whoosh.query.qcore import QueryError
|
|
25
|
+
|
|
26
|
+
|
|
27
|
+
class RobustPhrase(Phrase):
|
|
28
|
+
"""
|
|
29
|
+
Matches documents containing a given phrase.
|
|
30
|
+
"""
|
|
31
|
+
|
|
32
|
+
def matcher(self, searcher, context=None):
|
|
33
|
+
"""
|
|
34
|
+
Override the default to not raise error on match exception but simply return not found
|
|
35
|
+
Needed to handle phrase search in whoosh.fields.IDLIST
|
|
36
|
+
"""
|
|
37
|
+
try:
|
|
38
|
+
return super().matcher(searcher, context)
|
|
39
|
+
except QueryError:
|
|
40
|
+
return NullMatcher()
|
|
41
|
+
|
|
42
|
+
|
|
43
|
+
class EODAGQueryParser(QueryParser):
|
|
44
|
+
"""
|
|
45
|
+
A hand-written query parser built on modular plug-ins.
|
|
46
|
+
|
|
47
|
+
Override the default to include specific EODAG configuration
|
|
48
|
+
"""
|
|
49
|
+
|
|
50
|
+
def __init__(
|
|
51
|
+
self,
|
|
52
|
+
filters: List[str],
|
|
53
|
+
schema: Schema,
|
|
54
|
+
):
|
|
55
|
+
"""
|
|
56
|
+
EODAG QueryParser initialization
|
|
57
|
+
|
|
58
|
+
:param filters: list of fieldnames to filter on
|
|
59
|
+
:type filters: List[str]
|
|
60
|
+
:param schema: Whoosh Schema
|
|
61
|
+
:type schma: :class:whoosh.fields.Schema
|
|
62
|
+
"""
|
|
63
|
+
super().__init__(
|
|
64
|
+
None,
|
|
65
|
+
schema=schema,
|
|
66
|
+
plugins=[
|
|
67
|
+
plugins.SingleQuotePlugin(),
|
|
68
|
+
plugins.FieldsPlugin(),
|
|
69
|
+
plugins.WildcardPlugin(),
|
|
70
|
+
plugins.PhrasePlugin(),
|
|
71
|
+
plugins.GroupPlugin(),
|
|
72
|
+
plugins.OperatorsPlugin(),
|
|
73
|
+
plugins.BoostPlugin(),
|
|
74
|
+
plugins.EveryPlugin(),
|
|
75
|
+
plugins.RangePlugin(),
|
|
76
|
+
plugins.PlusMinusPlugin(),
|
|
77
|
+
plugins.MultifieldPlugin(filters, fieldboosts=None),
|
|
78
|
+
],
|
|
79
|
+
phraseclass=RobustPhrase,
|
|
80
|
+
group=OrGroup,
|
|
81
|
+
)
|
eodag/utils/__init__.py
CHANGED
|
@@ -20,6 +20,7 @@
|
|
|
20
20
|
Everything that does not fit into one of the specialised categories of utilities in
|
|
21
21
|
this package should go here
|
|
22
22
|
"""
|
|
23
|
+
|
|
23
24
|
from __future__ import annotations
|
|
24
25
|
|
|
25
26
|
import ast
|
|
@@ -29,15 +30,19 @@ import functools
|
|
|
29
30
|
import hashlib
|
|
30
31
|
import inspect
|
|
31
32
|
import logging as py_logging
|
|
33
|
+
import mimetypes
|
|
32
34
|
import os
|
|
33
35
|
import re
|
|
34
36
|
import shutil
|
|
37
|
+
import ssl
|
|
35
38
|
import string
|
|
39
|
+
import sys
|
|
36
40
|
import types
|
|
37
41
|
import unicodedata
|
|
38
42
|
import warnings
|
|
39
43
|
from collections import defaultdict
|
|
40
44
|
from copy import deepcopy as copy_deepcopy
|
|
45
|
+
from dataclasses import dataclass
|
|
41
46
|
from datetime import datetime as dt
|
|
42
47
|
from email.message import Message
|
|
43
48
|
from glob import glob
|
|
@@ -52,10 +57,12 @@ from typing import (
|
|
|
52
57
|
Dict,
|
|
53
58
|
Iterator,
|
|
54
59
|
List,
|
|
60
|
+
Mapping,
|
|
55
61
|
Optional,
|
|
56
62
|
Tuple,
|
|
57
63
|
Type,
|
|
58
64
|
Union,
|
|
65
|
+
cast,
|
|
59
66
|
)
|
|
60
67
|
|
|
61
68
|
# All modules using these should import them from utils package
|
|
@@ -71,12 +78,16 @@ from urllib.parse import ( # noqa; noqa
|
|
|
71
78
|
)
|
|
72
79
|
from urllib.request import url2pathname
|
|
73
80
|
|
|
74
|
-
|
|
81
|
+
if sys.version_info >= (3, 9):
|
|
75
82
|
from typing import Annotated, get_args, get_origin # noqa
|
|
76
|
-
|
|
77
|
-
# for python < 3.9
|
|
83
|
+
else:
|
|
78
84
|
from typing_extensions import Annotated, get_args, get_origin # type: ignore # noqa
|
|
79
85
|
|
|
86
|
+
if sys.version_info >= (3, 12):
|
|
87
|
+
from typing import Unpack # type: ignore # noqa
|
|
88
|
+
else:
|
|
89
|
+
from typing_extensions import Unpack # noqa
|
|
90
|
+
|
|
80
91
|
import click
|
|
81
92
|
import orjson
|
|
82
93
|
import shapefile
|
|
@@ -89,7 +100,7 @@ from jsonpath_ng.ext import parse
|
|
|
89
100
|
from jsonpath_ng.jsonpath import Child, Fields, Index, Root, Slice
|
|
90
101
|
from requests import HTTPError
|
|
91
102
|
from shapely.geometry import Polygon, shape
|
|
92
|
-
from shapely.geometry.base import BaseGeometry
|
|
103
|
+
from shapely.geometry.base import GEOMETRY_TYPES, BaseGeometry
|
|
93
104
|
from tqdm.auto import tqdm
|
|
94
105
|
|
|
95
106
|
from eodag.utils import logging as eodag_logging
|
|
@@ -128,6 +139,9 @@ DEFAULT_ITEMS_PER_PAGE = 20
|
|
|
128
139
|
# (DEFAULT_ITEMS_PER_PAGE) to increase it to the known and current minimum value (mundi)
|
|
129
140
|
DEFAULT_MAX_ITEMS_PER_PAGE = 50
|
|
130
141
|
|
|
142
|
+
# default product-types start date
|
|
143
|
+
DEFAULT_MISSION_START_DATE = "2015-01-01T00:00:00Z"
|
|
144
|
+
|
|
131
145
|
|
|
132
146
|
def _deprecated(reason: str = "", version: Optional[str] = None) -> Callable[..., Any]:
|
|
133
147
|
"""Simple decorator to mark functions/methods/classes as deprecated.
|
|
@@ -822,7 +836,7 @@ def list_items_recursive_apply(
|
|
|
822
836
|
|
|
823
837
|
|
|
824
838
|
def items_recursive_sort(
|
|
825
|
-
input_obj: Union[List[Any], Dict[Any, Any]]
|
|
839
|
+
input_obj: Union[List[Any], Dict[Any, Any]],
|
|
826
840
|
) -> Union[List[Any], Dict[Any, Any]]:
|
|
827
841
|
"""Recursive sort dict items contained in input object (dict or list)
|
|
828
842
|
|
|
@@ -1083,7 +1097,10 @@ def get_geometry_from_various(
|
|
|
1083
1097
|
geom_arg = query_args["geometry"]
|
|
1084
1098
|
|
|
1085
1099
|
bbox_keys = ["lonmin", "latmin", "lonmax", "latmax"]
|
|
1086
|
-
if isinstance(geom_arg, dict) and
|
|
1100
|
+
if isinstance(geom_arg, dict) and geom_arg.get("type") in GEOMETRY_TYPES:
|
|
1101
|
+
# geojson geometry
|
|
1102
|
+
geom = cast(BaseGeometry, shape(geom_arg))
|
|
1103
|
+
elif isinstance(geom_arg, dict) and all(k in geom_arg for k in bbox_keys):
|
|
1087
1104
|
# bbox dict
|
|
1088
1105
|
geom = Polygon(
|
|
1089
1106
|
(
|
|
@@ -1230,7 +1247,9 @@ def cached_parse(str_to_parse: str) -> JSONPath:
|
|
|
1230
1247
|
|
|
1231
1248
|
@functools.lru_cache()
|
|
1232
1249
|
def _mutable_cached_yaml_load(config_path: str) -> Any:
|
|
1233
|
-
with open(
|
|
1250
|
+
with open(
|
|
1251
|
+
os.path.abspath(os.path.realpath(config_path)), mode="r", encoding="utf-8"
|
|
1252
|
+
) as fh:
|
|
1234
1253
|
return yaml.load(fh, Loader=yaml.SafeLoader)
|
|
1235
1254
|
|
|
1236
1255
|
|
|
@@ -1289,8 +1308,9 @@ def get_bucket_name_and_prefix(
|
|
|
1289
1308
|
prefix = path
|
|
1290
1309
|
elif bucket_path_level is not None:
|
|
1291
1310
|
parts = path.split("/")
|
|
1292
|
-
bucket, prefix =
|
|
1293
|
-
parts[
|
|
1311
|
+
bucket, prefix = (
|
|
1312
|
+
parts[bucket_path_level],
|
|
1313
|
+
"/".join(parts[(bucket_path_level + 1) :]),
|
|
1294
1314
|
)
|
|
1295
1315
|
|
|
1296
1316
|
return bucket, prefix
|
|
@@ -1409,3 +1429,46 @@ def cast_scalar_value(value: Any, new_type: Any) -> Any:
|
|
|
1409
1429
|
return eval(value.capitalize())
|
|
1410
1430
|
|
|
1411
1431
|
return new_type(value)
|
|
1432
|
+
|
|
1433
|
+
|
|
1434
|
+
@dataclass
|
|
1435
|
+
class StreamResponse:
|
|
1436
|
+
"""Represents a streaming response"""
|
|
1437
|
+
|
|
1438
|
+
content: Iterator[bytes]
|
|
1439
|
+
headers: Optional[Mapping[str, str]] = None
|
|
1440
|
+
media_type: Optional[str] = None
|
|
1441
|
+
status_code: Optional[int] = None
|
|
1442
|
+
|
|
1443
|
+
|
|
1444
|
+
def guess_file_type(file: str) -> Optional[str]:
|
|
1445
|
+
"""guess the mime type of a file or URL based on its extension"""
|
|
1446
|
+
mimetypes.add_type("text/xml", ".xsd")
|
|
1447
|
+
mimetypes.add_type("application/x-grib", ".grib")
|
|
1448
|
+
mime_type, _ = mimetypes.guess_type(file, False)
|
|
1449
|
+
return mime_type
|
|
1450
|
+
|
|
1451
|
+
|
|
1452
|
+
def guess_extension(type: str) -> Optional[str]:
|
|
1453
|
+
"""guess extension from mime type"""
|
|
1454
|
+
mimetypes.add_type("text/xml", ".xsd")
|
|
1455
|
+
mimetypes.add_type("application/x-grib", ".grib")
|
|
1456
|
+
return mimetypes.guess_extension(type, strict=False)
|
|
1457
|
+
|
|
1458
|
+
|
|
1459
|
+
def get_ssl_context(ssl_verify: bool) -> ssl.SSLContext:
|
|
1460
|
+
"""
|
|
1461
|
+
Returns an SSL context based on ssl_verify argument.
|
|
1462
|
+
:param ssl_verify: ssl_verify parameter
|
|
1463
|
+
:type ssl_verify: bool
|
|
1464
|
+
:returns: An SSL context object.
|
|
1465
|
+
:rtype: ssl.SSLContext
|
|
1466
|
+
"""
|
|
1467
|
+
ctx = ssl.create_default_context()
|
|
1468
|
+
if not ssl_verify:
|
|
1469
|
+
ctx.check_hostname = False
|
|
1470
|
+
ctx.verify_mode = ssl.CERT_NONE
|
|
1471
|
+
else:
|
|
1472
|
+
ctx.check_hostname = True
|
|
1473
|
+
ctx.verify_mode = ssl.CERT_REQUIRED
|
|
1474
|
+
return ctx
|