eodag 3.0.1__py3-none-any.whl → 3.1.0b1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- eodag/api/core.py +116 -86
- eodag/api/product/_assets.py +6 -6
- eodag/api/product/_product.py +18 -18
- eodag/api/product/metadata_mapping.py +39 -11
- eodag/cli.py +22 -1
- eodag/config.py +14 -14
- eodag/plugins/apis/ecmwf.py +37 -14
- eodag/plugins/apis/usgs.py +5 -5
- eodag/plugins/authentication/openid_connect.py +2 -2
- eodag/plugins/authentication/token.py +37 -6
- eodag/plugins/crunch/filter_property.py +2 -3
- eodag/plugins/download/aws.py +11 -12
- eodag/plugins/download/base.py +30 -39
- eodag/plugins/download/creodias_s3.py +29 -0
- eodag/plugins/download/http.py +144 -152
- eodag/plugins/download/s3rest.py +5 -7
- eodag/plugins/search/base.py +73 -25
- eodag/plugins/search/build_search_result.py +1047 -310
- eodag/plugins/search/creodias_s3.py +25 -19
- eodag/plugins/search/data_request_search.py +1 -1
- eodag/plugins/search/qssearch.py +51 -139
- eodag/resources/ext_product_types.json +1 -1
- eodag/resources/product_types.yml +391 -32
- eodag/resources/providers.yml +678 -1744
- eodag/rest/core.py +92 -62
- eodag/rest/server.py +31 -4
- eodag/rest/types/eodag_search.py +6 -0
- eodag/rest/types/queryables.py +5 -6
- eodag/rest/utils/__init__.py +3 -0
- eodag/types/__init__.py +56 -15
- eodag/types/download_args.py +2 -2
- eodag/types/queryables.py +180 -72
- eodag/types/whoosh.py +126 -0
- eodag/utils/__init__.py +71 -10
- eodag/utils/exceptions.py +27 -20
- eodag/utils/repr.py +65 -6
- eodag/utils/requests.py +11 -11
- {eodag-3.0.1.dist-info → eodag-3.1.0b1.dist-info}/METADATA +76 -76
- {eodag-3.0.1.dist-info → eodag-3.1.0b1.dist-info}/RECORD +43 -44
- {eodag-3.0.1.dist-info → eodag-3.1.0b1.dist-info}/WHEEL +1 -1
- {eodag-3.0.1.dist-info → eodag-3.1.0b1.dist-info}/entry_points.txt +3 -2
- eodag/utils/constraints.py +0 -244
- {eodag-3.0.1.dist-info → eodag-3.1.0b1.dist-info}/LICENSE +0 -0
- {eodag-3.0.1.dist-info → eodag-3.1.0b1.dist-info}/top_level.txt +0 -0
eodag/types/queryables.py
CHANGED
|
@@ -15,11 +15,20 @@
|
|
|
15
15
|
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
16
16
|
# See the License for the specific language governing permissions and
|
|
17
17
|
# limitations under the License.
|
|
18
|
-
from
|
|
18
|
+
from __future__ import annotations
|
|
19
|
+
|
|
20
|
+
from collections import UserDict
|
|
21
|
+
from datetime import date, datetime
|
|
22
|
+
from typing import Annotated, Any, Optional, Union
|
|
19
23
|
|
|
20
24
|
from annotated_types import Lt
|
|
21
25
|
from pydantic import BaseModel, Field
|
|
26
|
+
from pydantic.fields import FieldInfo
|
|
22
27
|
from pydantic.types import PositiveInt
|
|
28
|
+
from pydantic_core import PydanticUndefined
|
|
29
|
+
|
|
30
|
+
from eodag.types import annotated_dict_to_model, model_fields_to_annotated
|
|
31
|
+
from eodag.utils.repr import remove_class_repr, shorter_type_repr
|
|
23
32
|
|
|
24
33
|
Percentage = Annotated[PositiveInt, Lt(100)]
|
|
25
34
|
|
|
@@ -28,17 +37,11 @@ class CommonQueryables(BaseModel):
|
|
|
28
37
|
"""A class representing search common queryable properties."""
|
|
29
38
|
|
|
30
39
|
productType: Annotated[str, Field()]
|
|
31
|
-
id: Annotated[Optional[str], Field(None)]
|
|
32
|
-
start: Annotated[Optional[str], Field(None, alias="startTimeFromAscendingNode")]
|
|
33
|
-
end: Annotated[Optional[str], Field(None, alias="completionTimeFromAscendingNode")]
|
|
34
|
-
geom: Annotated[Optional[str], Field(None, alias="geometry")]
|
|
35
40
|
|
|
36
41
|
@classmethod
|
|
37
42
|
def get_queryable_from_alias(cls, value: str) -> str:
|
|
38
43
|
"""Get queryable parameter from alias
|
|
39
44
|
|
|
40
|
-
>>> CommonQueryables.get_queryable_from_alias('startTimeFromAscendingNode')
|
|
41
|
-
'start'
|
|
42
45
|
>>> CommonQueryables.get_queryable_from_alias('productType')
|
|
43
46
|
'productType'
|
|
44
47
|
"""
|
|
@@ -49,75 +52,180 @@ class CommonQueryables(BaseModel):
|
|
|
49
52
|
}
|
|
50
53
|
return alias_map.get(value, value)
|
|
51
54
|
|
|
55
|
+
@classmethod
|
|
56
|
+
def get_with_default(
|
|
57
|
+
cls, field: str, default: Optional[Any]
|
|
58
|
+
) -> Annotated[Any, FieldInfo]:
|
|
59
|
+
"""Get field and set default value."""
|
|
60
|
+
annotated_fields = model_fields_to_annotated(cls.model_fields)
|
|
61
|
+
f = annotated_fields[field]
|
|
62
|
+
if default is None:
|
|
63
|
+
return f
|
|
64
|
+
f.__metadata__[0].default = default
|
|
65
|
+
return f
|
|
66
|
+
|
|
52
67
|
|
|
53
68
|
class Queryables(CommonQueryables):
|
|
54
|
-
"""A class representing all search queryable properties.
|
|
69
|
+
"""A class representing all search queryable properties.
|
|
55
70
|
|
|
56
|
-
|
|
71
|
+
Parameters default value is set to ``None`` to have them not required.
|
|
72
|
+
"""
|
|
73
|
+
|
|
74
|
+
start: Annotated[
|
|
75
|
+
Union[datetime, date], Field(None, alias="startTimeFromAscendingNode")
|
|
76
|
+
]
|
|
77
|
+
end: Annotated[
|
|
78
|
+
Union[datetime, date], Field(None, alias="completionTimeFromAscendingNode")
|
|
79
|
+
]
|
|
80
|
+
geom: Annotated[str, Field(None, alias="geometry")]
|
|
81
|
+
uid: Annotated[str, Field(None)]
|
|
57
82
|
# OpenSearch Parameters for Collection Search (Table 3)
|
|
58
|
-
doi: Annotated[
|
|
59
|
-
platform: Annotated[
|
|
60
|
-
platformSerialIdentifier: Annotated[
|
|
61
|
-
instrument: Annotated[
|
|
62
|
-
sensorType: Annotated[
|
|
63
|
-
compositeType: Annotated[
|
|
64
|
-
processingLevel: Annotated[
|
|
65
|
-
orbitType: Annotated[
|
|
66
|
-
spectralRange: Annotated[
|
|
67
|
-
wavelengths: Annotated[
|
|
68
|
-
hasSecurityConstraints: Annotated[
|
|
69
|
-
dissemination: Annotated[
|
|
83
|
+
doi: Annotated[str, Field(None)]
|
|
84
|
+
platform: Annotated[str, Field(None)]
|
|
85
|
+
platformSerialIdentifier: Annotated[str, Field(None)]
|
|
86
|
+
instrument: Annotated[str, Field(None)]
|
|
87
|
+
sensorType: Annotated[str, Field(None)]
|
|
88
|
+
compositeType: Annotated[str, Field(None)]
|
|
89
|
+
processingLevel: Annotated[str, Field(None)]
|
|
90
|
+
orbitType: Annotated[str, Field(None)]
|
|
91
|
+
spectralRange: Annotated[str, Field(None)]
|
|
92
|
+
wavelengths: Annotated[str, Field(None)]
|
|
93
|
+
hasSecurityConstraints: Annotated[str, Field(None)]
|
|
94
|
+
dissemination: Annotated[str, Field(None)]
|
|
70
95
|
# INSPIRE obligated OpenSearch Parameters for Collection Search (Table 4)
|
|
71
|
-
title: Annotated[
|
|
72
|
-
topicCategory: Annotated[
|
|
73
|
-
keyword: Annotated[
|
|
74
|
-
abstract: Annotated[
|
|
75
|
-
resolution: Annotated[
|
|
76
|
-
organisationName: Annotated[
|
|
77
|
-
organisationRole: Annotated[
|
|
78
|
-
publicationDate: Annotated[
|
|
79
|
-
lineage: Annotated[
|
|
80
|
-
useLimitation: Annotated[
|
|
81
|
-
accessConstraint: Annotated[
|
|
82
|
-
otherConstraint: Annotated[
|
|
83
|
-
classification: Annotated[
|
|
84
|
-
language: Annotated[
|
|
85
|
-
specification: Annotated[
|
|
96
|
+
title: Annotated[str, Field(None)]
|
|
97
|
+
topicCategory: Annotated[str, Field(None)]
|
|
98
|
+
keyword: Annotated[str, Field(None)]
|
|
99
|
+
abstract: Annotated[str, Field(None)]
|
|
100
|
+
resolution: Annotated[int, Field(None)]
|
|
101
|
+
organisationName: Annotated[str, Field(None)]
|
|
102
|
+
organisationRole: Annotated[str, Field(None)]
|
|
103
|
+
publicationDate: Annotated[str, Field(None)]
|
|
104
|
+
lineage: Annotated[str, Field(None)]
|
|
105
|
+
useLimitation: Annotated[str, Field(None)]
|
|
106
|
+
accessConstraint: Annotated[str, Field(None)]
|
|
107
|
+
otherConstraint: Annotated[str, Field(None)]
|
|
108
|
+
classification: Annotated[str, Field(None)]
|
|
109
|
+
language: Annotated[str, Field(None)]
|
|
110
|
+
specification: Annotated[str, Field(None)]
|
|
86
111
|
# OpenSearch Parameters for Product Search (Table 5)
|
|
87
|
-
parentIdentifier: Annotated[
|
|
88
|
-
productionStatus: Annotated[
|
|
89
|
-
acquisitionType: Annotated[
|
|
90
|
-
orbitNumber: Annotated[
|
|
91
|
-
orbitDirection: Annotated[
|
|
92
|
-
track: Annotated[
|
|
93
|
-
frame: Annotated[
|
|
94
|
-
swathIdentifier: Annotated[
|
|
95
|
-
cloudCover: Annotated[
|
|
96
|
-
snowCover: Annotated[
|
|
97
|
-
lowestLocation: Annotated[
|
|
98
|
-
highestLocation: Annotated[
|
|
99
|
-
productVersion: Annotated[
|
|
100
|
-
productQualityStatus: Annotated[
|
|
101
|
-
productQualityDegradationTag: Annotated[
|
|
102
|
-
processorName: Annotated[
|
|
103
|
-
processingCenter: Annotated[
|
|
104
|
-
creationDate: Annotated[
|
|
105
|
-
modificationDate: Annotated[
|
|
106
|
-
processingDate: Annotated[
|
|
107
|
-
sensorMode: Annotated[
|
|
108
|
-
archivingCenter: Annotated[
|
|
109
|
-
processingMode: Annotated[
|
|
112
|
+
parentIdentifier: Annotated[str, Field(None)]
|
|
113
|
+
productionStatus: Annotated[str, Field(None)]
|
|
114
|
+
acquisitionType: Annotated[str, Field(None)]
|
|
115
|
+
orbitNumber: Annotated[int, Field(None)]
|
|
116
|
+
orbitDirection: Annotated[str, Field(None)]
|
|
117
|
+
track: Annotated[str, Field(None)]
|
|
118
|
+
frame: Annotated[str, Field(None)]
|
|
119
|
+
swathIdentifier: Annotated[str, Field(None)]
|
|
120
|
+
cloudCover: Annotated[Percentage, Field(None)]
|
|
121
|
+
snowCover: Annotated[Percentage, Field(None)]
|
|
122
|
+
lowestLocation: Annotated[str, Field(None)]
|
|
123
|
+
highestLocation: Annotated[str, Field(None)]
|
|
124
|
+
productVersion: Annotated[str, Field(None)]
|
|
125
|
+
productQualityStatus: Annotated[str, Field(None)]
|
|
126
|
+
productQualityDegradationTag: Annotated[str, Field(None)]
|
|
127
|
+
processorName: Annotated[str, Field(None)]
|
|
128
|
+
processingCenter: Annotated[str, Field(None)]
|
|
129
|
+
creationDate: Annotated[str, Field(None)]
|
|
130
|
+
modificationDate: Annotated[str, Field(None)]
|
|
131
|
+
processingDate: Annotated[str, Field(None)]
|
|
132
|
+
sensorMode: Annotated[str, Field(None)]
|
|
133
|
+
archivingCenter: Annotated[str, Field(None)]
|
|
134
|
+
processingMode: Annotated[str, Field(None)]
|
|
110
135
|
# OpenSearch Parameters for Acquistion Parameters Search (Table 6)
|
|
111
|
-
availabilityTime: Annotated[
|
|
112
|
-
acquisitionStation: Annotated[
|
|
113
|
-
acquisitionSubType: Annotated[
|
|
114
|
-
illuminationAzimuthAngle: Annotated[
|
|
115
|
-
illuminationZenithAngle: Annotated[
|
|
116
|
-
illuminationElevationAngle: Annotated[
|
|
117
|
-
polarizationMode: Annotated[
|
|
118
|
-
polarizationChannels: Annotated[
|
|
119
|
-
antennaLookDirection: Annotated[
|
|
120
|
-
minimumIncidenceAngle: Annotated[
|
|
121
|
-
maximumIncidenceAngle: Annotated[
|
|
122
|
-
dopplerFrequency: Annotated[
|
|
123
|
-
incidenceAngleVariation: Annotated[
|
|
136
|
+
availabilityTime: Annotated[str, Field(None)]
|
|
137
|
+
acquisitionStation: Annotated[str, Field(None)]
|
|
138
|
+
acquisitionSubType: Annotated[str, Field(None)]
|
|
139
|
+
illuminationAzimuthAngle: Annotated[str, Field(None)]
|
|
140
|
+
illuminationZenithAngle: Annotated[str, Field(None)]
|
|
141
|
+
illuminationElevationAngle: Annotated[str, Field(None)]
|
|
142
|
+
polarizationMode: Annotated[str, Field(None)]
|
|
143
|
+
polarizationChannels: Annotated[str, Field(None)]
|
|
144
|
+
antennaLookDirection: Annotated[str, Field(None)]
|
|
145
|
+
minimumIncidenceAngle: Annotated[float, Field(None)]
|
|
146
|
+
maximumIncidenceAngle: Annotated[float, Field(None)]
|
|
147
|
+
dopplerFrequency: Annotated[float, Field(None)]
|
|
148
|
+
incidenceAngleVariation: Annotated[float, Field(None)]
|
|
149
|
+
|
|
150
|
+
|
|
151
|
+
class QueryablesDict(UserDict):
|
|
152
|
+
"""Class inheriting from UserDict which contains queryables with their annotated type;
|
|
153
|
+
|
|
154
|
+
:param additional_properties: if additional properties (properties not given in EODAG config)
|
|
155
|
+
are allowed
|
|
156
|
+
:param kwargs: named arguments to initialise the dict (queryable keys + annotated types)
|
|
157
|
+
"""
|
|
158
|
+
|
|
159
|
+
additional_properties: bool = Field(True)
|
|
160
|
+
additional_information: str = Field("")
|
|
161
|
+
|
|
162
|
+
def __init__(
|
|
163
|
+
self, additional_properties: bool, additional_information: str = "", **kwargs
|
|
164
|
+
):
|
|
165
|
+
self.additional_properties = additional_properties
|
|
166
|
+
self.additional_information = additional_information
|
|
167
|
+
super().__init__(kwargs)
|
|
168
|
+
|
|
169
|
+
def _repr_html_(self, embedded: bool = False) -> str:
|
|
170
|
+
add_info = (
|
|
171
|
+
f" additional_information={self.additional_information}"
|
|
172
|
+
if self.additional_information
|
|
173
|
+
else ""
|
|
174
|
+
)
|
|
175
|
+
thead = (
|
|
176
|
+
f"""<thead><tr><td style='text-align: left; color: grey;'>
|
|
177
|
+
{type(self).__name__} ({len(self)}) - additional_properties={
|
|
178
|
+
self.additional_properties}
|
|
179
|
+
"""
|
|
180
|
+
+ add_info
|
|
181
|
+
+ "</td></tr></thead>"
|
|
182
|
+
if not embedded
|
|
183
|
+
else ""
|
|
184
|
+
)
|
|
185
|
+
tr_style = "style='background-color: transparent;'" if embedded else ""
|
|
186
|
+
return (
|
|
187
|
+
f"<table>{thead}<tbody>"
|
|
188
|
+
+ "".join(
|
|
189
|
+
[
|
|
190
|
+
f"""<tr {tr_style}><td style='text-align: left;'>
|
|
191
|
+
<details><summary style='color: grey;'>
|
|
192
|
+
<span style='color: black'>'{k}'</span>: 
|
|
193
|
+
typing.Annotated[{
|
|
194
|
+
"<span style='color: black'>" + shorter_type_repr(v.__args__[0]) + "</span>, "
|
|
195
|
+
}
|
|
196
|
+
FieldInfo({"'default': '<span style='color: black'>"
|
|
197
|
+
+ str(v.__metadata__[0].get_default()) + "</span>', "
|
|
198
|
+
if v.__metadata__[0].get_default()
|
|
199
|
+
and v.__metadata__[0].get_default() != PydanticUndefined else ""}
|
|
200
|
+
{"'required': <span style='color: black'>"
|
|
201
|
+
+ str(v.__metadata__[0].is_required()) + "</span>,"}
|
|
202
|
+
...
|
|
203
|
+
)]
|
|
204
|
+
</summary>
|
|
205
|
+
<span style='color: grey'>typing.Annotated[</span><table style='margin: 0;'>
|
|
206
|
+
<tr style='background-color: transparent;'>
|
|
207
|
+
<td style='padding: 5px 0 0 10px; text-align: left; vertical-align:top;'>
|
|
208
|
+
{remove_class_repr(str(v.__args__[0]))},</td>
|
|
209
|
+
</tr><tr style='background-color: transparent;'>
|
|
210
|
+
<td style='padding: 5px 0 0 10px; text-align: left; vertical-align:top;'>
|
|
211
|
+
{v.__metadata__[0].__repr__()}</td>
|
|
212
|
+
</tr>
|
|
213
|
+
</table><span style='color: grey'>]</span>
|
|
214
|
+
</details>
|
|
215
|
+
</td></tr>
|
|
216
|
+
"""
|
|
217
|
+
for k, v in self.items()
|
|
218
|
+
]
|
|
219
|
+
)
|
|
220
|
+
+ "</tbody></table>"
|
|
221
|
+
)
|
|
222
|
+
|
|
223
|
+
def get_model(self, model_name: str = "Queryables") -> BaseModel:
|
|
224
|
+
"""
|
|
225
|
+
Converts object from :class:`eodag.api.product.QueryablesDict` to :class:`pydantic.BaseModel`
|
|
226
|
+
so that validation can be performed
|
|
227
|
+
|
|
228
|
+
:param model_name: name used for :class:`pydantic.BaseModel` creation
|
|
229
|
+
:return: pydantic BaseModel of the queryables dict
|
|
230
|
+
"""
|
|
231
|
+
return annotated_dict_to_model(model_name, self.data)
|
eodag/types/whoosh.py
CHANGED
|
@@ -18,10 +18,13 @@
|
|
|
18
18
|
from typing import List
|
|
19
19
|
|
|
20
20
|
from whoosh.fields import Schema
|
|
21
|
+
from whoosh.index import _DEF_INDEX_NAME, FileIndex
|
|
21
22
|
from whoosh.matching import NullMatcher
|
|
22
23
|
from whoosh.qparser import OrGroup, QueryParser, plugins
|
|
23
24
|
from whoosh.query.positional import Phrase
|
|
24
25
|
from whoosh.query.qcore import QueryError
|
|
26
|
+
from whoosh.util.text import utf8encode
|
|
27
|
+
from whoosh.writing import SegmentWriter
|
|
25
28
|
|
|
26
29
|
|
|
27
30
|
class RobustPhrase(Phrase):
|
|
@@ -77,3 +80,126 @@ class EODAGQueryParser(QueryParser):
|
|
|
77
80
|
phraseclass=RobustPhrase,
|
|
78
81
|
group=OrGroup,
|
|
79
82
|
)
|
|
83
|
+
|
|
84
|
+
|
|
85
|
+
class CleanSegmentWriter(SegmentWriter):
|
|
86
|
+
"""Override to clean up writer for failed document add when exceptions were absorbed
|
|
87
|
+
cf: https://github.com/whoosh-community/whoosh/pull/543
|
|
88
|
+
"""
|
|
89
|
+
|
|
90
|
+
def add_document(self, **fields):
|
|
91
|
+
"""Add document"""
|
|
92
|
+
self._check_state()
|
|
93
|
+
perdocwriter = self.perdocwriter
|
|
94
|
+
schema = self.schema
|
|
95
|
+
docnum = self.docnum
|
|
96
|
+
add_post = self.pool.add
|
|
97
|
+
|
|
98
|
+
docboost = self._doc_boost(fields)
|
|
99
|
+
fieldnames = sorted(
|
|
100
|
+
[name for name in fields.keys() if not name.startswith("_")]
|
|
101
|
+
)
|
|
102
|
+
self._check_fields(schema, fieldnames)
|
|
103
|
+
|
|
104
|
+
perdocwriter.start_doc(docnum)
|
|
105
|
+
|
|
106
|
+
try:
|
|
107
|
+
for fieldname in fieldnames:
|
|
108
|
+
value = fields.get(fieldname)
|
|
109
|
+
if value is None:
|
|
110
|
+
continue
|
|
111
|
+
field = schema[fieldname]
|
|
112
|
+
|
|
113
|
+
length = 0
|
|
114
|
+
if field.indexed:
|
|
115
|
+
# TODO: Method for adding progressive field values, ie
|
|
116
|
+
# setting start_pos/start_char?
|
|
117
|
+
fieldboost = self._field_boost(fields, fieldname, docboost)
|
|
118
|
+
# Ask the field to return a list of (text, weight, vbytes)
|
|
119
|
+
# tuples
|
|
120
|
+
items = field.index(value)
|
|
121
|
+
# Only store the length if the field is marked scorable
|
|
122
|
+
scorable = field.scorable
|
|
123
|
+
# Add the terms to the pool
|
|
124
|
+
for tbytes, freq, weight, vbytes in items:
|
|
125
|
+
weight *= fieldboost
|
|
126
|
+
if scorable:
|
|
127
|
+
length += freq
|
|
128
|
+
add_post((fieldname, tbytes, docnum, weight, vbytes))
|
|
129
|
+
|
|
130
|
+
if field.separate_spelling():
|
|
131
|
+
spellfield = field.spelling_fieldname(fieldname)
|
|
132
|
+
for word in field.spellable_words(value):
|
|
133
|
+
word = utf8encode(word)[0]
|
|
134
|
+
add_post((spellfield, word, 0, 1, vbytes))
|
|
135
|
+
|
|
136
|
+
vformat = field.vector
|
|
137
|
+
if vformat:
|
|
138
|
+
analyzer = field.analyzer
|
|
139
|
+
# Call the format's word_values method to get posting values
|
|
140
|
+
vitems = vformat.word_values(value, analyzer, mode="index")
|
|
141
|
+
# Remove unused frequency field from the tuple
|
|
142
|
+
vitems = sorted(
|
|
143
|
+
(text, weight, vbytes) for text, _, weight, vbytes in vitems
|
|
144
|
+
)
|
|
145
|
+
perdocwriter.add_vector_items(fieldname, field, vitems)
|
|
146
|
+
|
|
147
|
+
# Allow a custom value for stored field/column
|
|
148
|
+
customval = fields.get("_stored_%s" % fieldname, value)
|
|
149
|
+
|
|
150
|
+
# Add the stored value and length for this field to the per-
|
|
151
|
+
# document writer
|
|
152
|
+
sv = customval if field.stored else None
|
|
153
|
+
perdocwriter.add_field(fieldname, field, sv, length)
|
|
154
|
+
|
|
155
|
+
column = field.column_type
|
|
156
|
+
if column and customval is not None:
|
|
157
|
+
cv = field.to_column_value(customval)
|
|
158
|
+
perdocwriter.add_column_value(fieldname, column, cv)
|
|
159
|
+
except Exception as ex:
|
|
160
|
+
# cancel doc
|
|
161
|
+
perdocwriter._doccount -= 1
|
|
162
|
+
perdocwriter._indoc = False
|
|
163
|
+
raise ex
|
|
164
|
+
|
|
165
|
+
perdocwriter.finish_doc()
|
|
166
|
+
self._added = True
|
|
167
|
+
self.docnum += 1
|
|
168
|
+
|
|
169
|
+
|
|
170
|
+
class CleanFileIndex(FileIndex):
|
|
171
|
+
"""Override to call CleanSegmentWriter"""
|
|
172
|
+
|
|
173
|
+
def writer(self, procs=1, **kwargs):
|
|
174
|
+
"""file index writer"""
|
|
175
|
+
if procs > 1:
|
|
176
|
+
from whoosh.multiproc import MpWriter
|
|
177
|
+
|
|
178
|
+
return MpWriter(self, procs=procs, **kwargs)
|
|
179
|
+
else:
|
|
180
|
+
return CleanSegmentWriter(self, **kwargs)
|
|
181
|
+
|
|
182
|
+
|
|
183
|
+
def create_in(dirname, schema, indexname=None):
|
|
184
|
+
"""
|
|
185
|
+
Override to call the CleanFileIndex.
|
|
186
|
+
|
|
187
|
+
Convenience function to create an index in a directory. Takes care of
|
|
188
|
+
creating a FileStorage object for you.
|
|
189
|
+
|
|
190
|
+
:param dirname: the path string of the directory in which to create the
|
|
191
|
+
index.
|
|
192
|
+
:param schema: a :class:`whoosh.fields.Schema` object describing the
|
|
193
|
+
index's fields.
|
|
194
|
+
:param indexname: the name of the index to create; you only need to specify
|
|
195
|
+
this if you are creating multiple indexes within the same storage
|
|
196
|
+
object.
|
|
197
|
+
:returns: :class:`Index`
|
|
198
|
+
"""
|
|
199
|
+
|
|
200
|
+
from whoosh.filedb.filestore import FileStorage
|
|
201
|
+
|
|
202
|
+
if not indexname:
|
|
203
|
+
indexname = _DEF_INDEX_NAME
|
|
204
|
+
storage = FileStorage(dirname)
|
|
205
|
+
return CleanFileIndex.create(storage, schema, indexname)
|
eodag/utils/__init__.py
CHANGED
|
@@ -84,6 +84,7 @@ if sys.version_info >= (3, 12):
|
|
|
84
84
|
else:
|
|
85
85
|
from typing_extensions import Unpack # noqa
|
|
86
86
|
|
|
87
|
+
|
|
87
88
|
import click
|
|
88
89
|
import orjson
|
|
89
90
|
import shapefile
|
|
@@ -125,8 +126,8 @@ REQ_RETRY_BACKOFF_FACTOR = 2
|
|
|
125
126
|
REQ_RETRY_STATUS_FORCELIST = [401, 429, 500, 502, 503, 504]
|
|
126
127
|
|
|
127
128
|
# default wait times in minutes
|
|
128
|
-
DEFAULT_DOWNLOAD_WAIT = 2 # in minutes
|
|
129
|
-
DEFAULT_DOWNLOAD_TIMEOUT =
|
|
129
|
+
DEFAULT_DOWNLOAD_WAIT = 0.2 # in minutes
|
|
130
|
+
DEFAULT_DOWNLOAD_TIMEOUT = 10 # in minutes
|
|
130
131
|
|
|
131
132
|
JSONPATH_MATCH = re.compile(r"^[\{\(]*\$(\..*)*$")
|
|
132
133
|
WORKABLE_JSONPATH_MATCH = re.compile(r"^\$(\.[a-zA-Z0-9-_:\.\[\]\"\(\)=\?\*]+)*$")
|
|
@@ -142,6 +143,13 @@ DEFAULT_MAX_ITEMS_PER_PAGE = 50
|
|
|
142
143
|
# default product-types start date
|
|
143
144
|
DEFAULT_MISSION_START_DATE = "2015-01-01T00:00:00Z"
|
|
144
145
|
|
|
146
|
+
# update missing mimetypes
|
|
147
|
+
mimetypes.add_type("text/xml", ".xsd")
|
|
148
|
+
mimetypes.add_type("application/x-grib", ".grib")
|
|
149
|
+
mimetypes.add_type("application/x-grib2", ".grib2")
|
|
150
|
+
# jp2 is missing on windows
|
|
151
|
+
mimetypes.add_type("image/jp2", ".jp2")
|
|
152
|
+
|
|
145
153
|
|
|
146
154
|
def _deprecated(reason: str = "", version: Optional[str] = None) -> Callable[..., Any]:
|
|
147
155
|
"""Simple decorator to mark functions/methods/classes as deprecated.
|
|
@@ -430,6 +438,33 @@ def datetime_range(start: dt, end: dt) -> Iterator[dt]:
|
|
|
430
438
|
yield start + datetime.timedelta(days=nday)
|
|
431
439
|
|
|
432
440
|
|
|
441
|
+
def is_range_in_range(valid_range: str, check_range: str) -> bool:
|
|
442
|
+
"""Check if the check_range is completely within the valid_range.
|
|
443
|
+
|
|
444
|
+
This function checks if both the start and end dates of the check_range
|
|
445
|
+
are within the start and end dates of the valid_range.
|
|
446
|
+
|
|
447
|
+
:param valid_range: The valid date range in the format 'YYYY-MM-DD/YYYY-MM-DD'.
|
|
448
|
+
:param check_range: The date range to check in the format 'YYYY-MM-DD/YYYY-MM-DD'.
|
|
449
|
+
:returns: True if check_range is within valid_range, otherwise False.
|
|
450
|
+
"""
|
|
451
|
+
if "/" not in valid_range or "/" not in check_range:
|
|
452
|
+
return False
|
|
453
|
+
|
|
454
|
+
# Split the date ranges into start and end dates
|
|
455
|
+
start_valid, end_valid = valid_range.split("/")
|
|
456
|
+
start_check, end_check = check_range.split("/")
|
|
457
|
+
|
|
458
|
+
# Convert the strings to datetime objects using fromisoformat
|
|
459
|
+
start_valid_dt = datetime.datetime.fromisoformat(start_valid)
|
|
460
|
+
end_valid_dt = datetime.datetime.fromisoformat(end_valid)
|
|
461
|
+
start_check_dt = datetime.datetime.fromisoformat(start_check)
|
|
462
|
+
end_check_dt = datetime.datetime.fromisoformat(end_check)
|
|
463
|
+
|
|
464
|
+
# Check if check_range is within valid_range
|
|
465
|
+
return start_valid_dt <= start_check_dt and end_valid_dt >= end_check_dt
|
|
466
|
+
|
|
467
|
+
|
|
433
468
|
class DownloadedCallback:
|
|
434
469
|
"""Example class for callback after each download in :meth:`~eodag.api.core.EODataAccessGateway.download_all`"""
|
|
435
470
|
|
|
@@ -1120,7 +1155,7 @@ def get_geometry_from_various(
|
|
|
1120
1155
|
class MockResponse:
|
|
1121
1156
|
"""Fake requests response"""
|
|
1122
1157
|
|
|
1123
|
-
def __init__(self, json_data: Any, status_code: int) -> None:
|
|
1158
|
+
def __init__(self, json_data: Any = None, status_code: int = 200) -> None:
|
|
1124
1159
|
self.json_data = json_data
|
|
1125
1160
|
self.status_code = status_code
|
|
1126
1161
|
self.content = json_data
|
|
@@ -1129,10 +1164,21 @@ class MockResponse:
|
|
|
1129
1164
|
"""Return json data"""
|
|
1130
1165
|
return self.json_data
|
|
1131
1166
|
|
|
1167
|
+
def __iter__(self):
|
|
1168
|
+
yield self
|
|
1169
|
+
|
|
1170
|
+
def __enter__(self):
|
|
1171
|
+
return self
|
|
1172
|
+
|
|
1173
|
+
def __exit__(self, exc_type, exc_val, exc_tb):
|
|
1174
|
+
pass
|
|
1175
|
+
|
|
1132
1176
|
def raise_for_status(self) -> None:
|
|
1133
1177
|
"""raises an exception when the status is not ok"""
|
|
1134
1178
|
if self.status_code != 200:
|
|
1135
|
-
|
|
1179
|
+
response = Response()
|
|
1180
|
+
response.status_code = self.status_code
|
|
1181
|
+
raise HTTPError(response=response)
|
|
1136
1182
|
|
|
1137
1183
|
|
|
1138
1184
|
def md5sum(file_path: str) -> str:
|
|
@@ -1373,17 +1419,32 @@ class StreamResponse:
|
|
|
1373
1419
|
|
|
1374
1420
|
|
|
1375
1421
|
def guess_file_type(file: str) -> Optional[str]:
|
|
1376
|
-
"""
|
|
1377
|
-
mimetypes
|
|
1378
|
-
|
|
1422
|
+
"""Guess the mime type of a file or URL based on its extension,
|
|
1423
|
+
using eodag extended mimetypes definition
|
|
1424
|
+
|
|
1425
|
+
>>> guess_file_type('foo.tiff')
|
|
1426
|
+
'image/tiff'
|
|
1427
|
+
>>> guess_file_type('foo.grib')
|
|
1428
|
+
'application/x-grib'
|
|
1429
|
+
|
|
1430
|
+
:param file: file url or path
|
|
1431
|
+
:returns: guessed mime type
|
|
1432
|
+
"""
|
|
1379
1433
|
mime_type, _ = mimetypes.guess_type(file, False)
|
|
1380
1434
|
return mime_type
|
|
1381
1435
|
|
|
1382
1436
|
|
|
1383
1437
|
def guess_extension(type: str) -> Optional[str]:
|
|
1384
|
-
"""
|
|
1385
|
-
|
|
1386
|
-
|
|
1438
|
+
"""Guess extension from mime type, using eodag extended mimetypes definition
|
|
1439
|
+
|
|
1440
|
+
>>> guess_extension('image/tiff')
|
|
1441
|
+
'.tiff'
|
|
1442
|
+
>>> guess_extension('application/x-grib')
|
|
1443
|
+
'.grib'
|
|
1444
|
+
|
|
1445
|
+
:param type: mime type
|
|
1446
|
+
:returns: guessed file extension
|
|
1447
|
+
"""
|
|
1387
1448
|
return mimetypes.guess_extension(type, strict=False)
|
|
1388
1449
|
|
|
1389
1450
|
|
eodag/utils/exceptions.py
CHANGED
|
@@ -22,21 +22,13 @@ from typing import TYPE_CHECKING, Annotated
|
|
|
22
22
|
if TYPE_CHECKING:
|
|
23
23
|
from typing import Optional, Set
|
|
24
24
|
|
|
25
|
-
from typing_extensions import Doc
|
|
25
|
+
from typing_extensions import Doc, Self
|
|
26
26
|
|
|
27
27
|
|
|
28
28
|
class EodagError(Exception):
|
|
29
29
|
"""General EODAG error"""
|
|
30
30
|
|
|
31
31
|
|
|
32
|
-
class ValidationError(EodagError):
|
|
33
|
-
"""Error validating data"""
|
|
34
|
-
|
|
35
|
-
def __init__(self, message: str, parameters: Set[str] = set()) -> None:
|
|
36
|
-
self.message = message
|
|
37
|
-
self.parameters = parameters
|
|
38
|
-
|
|
39
|
-
|
|
40
32
|
class PluginNotFoundError(EodagError):
|
|
41
33
|
"""Error when looking for a plugin class that was not defined"""
|
|
42
34
|
|
|
@@ -74,14 +66,19 @@ class AuthenticationError(EodagError):
|
|
|
74
66
|
authenticating a user"""
|
|
75
67
|
|
|
76
68
|
|
|
77
|
-
class DownloadError(EodagError):
|
|
78
|
-
"""An error indicating something wrong with the download process"""
|
|
79
|
-
|
|
80
|
-
|
|
81
69
|
class NotAvailableError(EodagError):
|
|
82
70
|
"""An error indicating that the product is not available for download"""
|
|
83
71
|
|
|
84
72
|
|
|
73
|
+
class NoMatchingProductType(EodagError):
|
|
74
|
+
"""An error indicating that eodag was unable to derive a product type from a set
|
|
75
|
+
of search parameters"""
|
|
76
|
+
|
|
77
|
+
|
|
78
|
+
class STACOpenerError(EodagError):
|
|
79
|
+
"""An error indicating that a STAC file could not be opened"""
|
|
80
|
+
|
|
81
|
+
|
|
85
82
|
class RequestError(EodagError):
|
|
86
83
|
"""An error indicating that a request has failed. Usually eodag functions
|
|
87
84
|
and methods should catch and skip this"""
|
|
@@ -89,7 +86,7 @@ class RequestError(EodagError):
|
|
|
89
86
|
status_code: Annotated[Optional[int], Doc("HTTP status code")] = None
|
|
90
87
|
|
|
91
88
|
@classmethod
|
|
92
|
-
def from_error(cls, error: Exception, msg: Optional[str] = None):
|
|
89
|
+
def from_error(cls, error: Exception, msg: Optional[str] = None) -> Self:
|
|
93
90
|
"""Generate a RequestError from an Exception"""
|
|
94
91
|
status_code = getattr(error, "code", None)
|
|
95
92
|
text = getattr(error, "msg", None)
|
|
@@ -99,7 +96,7 @@ class RequestError(EodagError):
|
|
|
99
96
|
# have a status code other than 200
|
|
100
97
|
if response is not None:
|
|
101
98
|
status_code = response.status_code
|
|
102
|
-
text = response.text
|
|
99
|
+
text = " ".join([text or "", response.text])
|
|
103
100
|
|
|
104
101
|
text = text or str(error)
|
|
105
102
|
|
|
@@ -108,13 +105,23 @@ class RequestError(EodagError):
|
|
|
108
105
|
return e
|
|
109
106
|
|
|
110
107
|
|
|
111
|
-
class
|
|
112
|
-
"""
|
|
113
|
-
|
|
108
|
+
class ValidationError(RequestError):
|
|
109
|
+
"""Error validating data"""
|
|
110
|
+
|
|
111
|
+
def __init__(self, message: str, parameters: Set[str] = set()) -> None:
|
|
112
|
+
self.message = message
|
|
113
|
+
self.parameters = parameters
|
|
114
|
+
|
|
115
|
+
@classmethod
|
|
116
|
+
def from_error(cls, error: Exception, msg: Optional[str] = None) -> Self:
|
|
117
|
+
"""Override parent from_error to handle ValidationError specificities."""
|
|
118
|
+
setattr(error, "msg", msg)
|
|
119
|
+
validation_error = super().from_error(error)
|
|
120
|
+
return validation_error
|
|
114
121
|
|
|
115
122
|
|
|
116
|
-
class
|
|
117
|
-
"""An error indicating
|
|
123
|
+
class DownloadError(RequestError):
|
|
124
|
+
"""An error indicating something wrong with the download process"""
|
|
118
125
|
|
|
119
126
|
|
|
120
127
|
class TimeOutError(RequestError):
|