sentinelhub 3.11.0__py3-none-any.whl → 3.11.2__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- sentinelhub/__init__.py +0 -7
- sentinelhub/_version.py +1 -1
- sentinelhub/api/__init__.py +0 -6
- sentinelhub/api/batch/__init__.py +1 -4
- sentinelhub/api/batch/base.py +1 -4
- sentinelhub/api/batch/process.py +159 -410
- sentinelhub/api/batch/process_v2.py +9 -320
- sentinelhub/api/batch/utils.py +3 -130
- sentinelhub/areas.py +2 -78
- sentinelhub/config.py +2 -2
- {sentinelhub-3.11.0.dist-info → sentinelhub-3.11.2.dist-info}/METADATA +3 -5
- {sentinelhub-3.11.0.dist-info → sentinelhub-3.11.2.dist-info}/RECORD +15 -15
- {sentinelhub-3.11.0.dist-info → sentinelhub-3.11.2.dist-info}/WHEEL +1 -1
- {sentinelhub-3.11.0.dist-info → sentinelhub-3.11.2.dist-info}/entry_points.txt +0 -0
- {sentinelhub-3.11.0.dist-info → sentinelhub-3.11.2.dist-info}/licenses/LICENSE.md +0 -0
|
@@ -1,325 +1,14 @@
|
|
|
1
1
|
"""
|
|
2
|
-
|
|
3
|
-
`Batch Process V2 <https://docs.sentinel-hub.com/api/latest/api/batchv2/>`__.
|
|
2
|
+
Deprecated module for Batch Processing v2 API. The contents have been moved to `sentinelhub.api.batch.process`.
|
|
4
3
|
"""
|
|
5
4
|
|
|
6
|
-
|
|
7
|
-
# do not use `from __future__ import annotations`, it clashes with `dataclass_json`
|
|
8
|
-
import datetime as dt
|
|
9
|
-
import logging
|
|
10
|
-
from dataclasses import dataclass, field
|
|
11
|
-
from typing import Any, Dict, Iterator, Optional, Union
|
|
5
|
+
import warnings
|
|
12
6
|
|
|
13
|
-
from
|
|
14
|
-
from
|
|
15
|
-
from typing_extensions import Literal
|
|
7
|
+
from sentinelhub.api.batch.process import * # noqa: F403 # pylint: disable=unused-wildcard-import, wildcard-import
|
|
8
|
+
from sentinelhub.exceptions import SHDeprecationWarning
|
|
16
9
|
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
from .base import BaseBatchClient, BaseBatchRequest, BatchRequestStatus, BatchUserAction, StoppedStatusReason
|
|
23
|
-
|
|
24
|
-
LOGGER = logging.getLogger(__name__)
|
|
25
|
-
|
|
26
|
-
BatchRequestType = Union[str, dict, "BatchProcessRequest"]
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
class BatchProcessClient(BaseBatchClient):
|
|
30
|
-
"""An interface class for Sentinel Hub Batch API version 2.
|
|
31
|
-
|
|
32
|
-
`Batch Process API <https://docs.sentinel-hub.com/api/latest/api/batchv2/>`__
|
|
33
|
-
"""
|
|
34
|
-
|
|
35
|
-
s3_specification = staticmethod(s3_specification)
|
|
36
|
-
|
|
37
|
-
# pylint: disable=too-many-public-methods
|
|
38
|
-
@staticmethod
|
|
39
|
-
def _get_service_url(base_url: str) -> str:
|
|
40
|
-
"""Provides URL to Catalog API"""
|
|
41
|
-
return f"{base_url}/api/v2/batch"
|
|
42
|
-
|
|
43
|
-
def _get_processing_url(self, request_id: Optional[str] = None) -> str:
|
|
44
|
-
"""Creates a URL for process endpoint"""
|
|
45
|
-
url = f"{self.service_url}/process"
|
|
46
|
-
if request_id is None:
|
|
47
|
-
return url
|
|
48
|
-
return f"{url}/{request_id}"
|
|
49
|
-
|
|
50
|
-
def create(
|
|
51
|
-
self,
|
|
52
|
-
process_request: Union[SentinelHubRequest, JsonDict],
|
|
53
|
-
input: Dict[str, Any], # noqa: A002 #pylint: disable=redefined-builtin
|
|
54
|
-
output: Dict[str, Any],
|
|
55
|
-
instance_type: Literal["normal", "large"] = "normal",
|
|
56
|
-
description: Optional[str] = None,
|
|
57
|
-
**kwargs: Any,
|
|
58
|
-
) -> "BatchProcessRequest":
|
|
59
|
-
"""Create a new batch request
|
|
60
|
-
|
|
61
|
-
`Batch Process V2 <https://docs.sentinel-hub.com/api/latest/api/batchv2/>`__
|
|
62
|
-
|
|
63
|
-
:param process_request: An instance of SentinelHubRequest class containing all request parameters.
|
|
64
|
-
Alternatively, it can also be just a payload dictionary for Process API request
|
|
65
|
-
:param input: A dictionary with input parameters. It can be built with `tiling_grid_input` or `geopackage_input`
|
|
66
|
-
methods.
|
|
67
|
-
:param output: A dictionary with output parameters. It can be built with `raster_output` or `zarr_output`
|
|
68
|
-
methods.
|
|
69
|
-
:param instance_type": Specifies which size of instances to use for the request.
|
|
70
|
-
:param description: A description of a batch request
|
|
71
|
-
:param kwargs: Any other arguments to be added to a dictionary of parameters.
|
|
72
|
-
"""
|
|
73
|
-
|
|
74
|
-
if isinstance(process_request, SentinelHubRequest):
|
|
75
|
-
request_dict = process_request.download_list[0].post_values
|
|
76
|
-
else:
|
|
77
|
-
request_dict = process_request
|
|
78
|
-
|
|
79
|
-
if not isinstance(request_dict, dict):
|
|
80
|
-
raise ValueError(
|
|
81
|
-
"Parameter sentinelhub_request should be an instance of SentinelHubRequest or a "
|
|
82
|
-
"dictionary with a request payload"
|
|
83
|
-
)
|
|
84
|
-
|
|
85
|
-
payload = remove_undefined(
|
|
86
|
-
{
|
|
87
|
-
"processRequest": request_dict,
|
|
88
|
-
"input": input,
|
|
89
|
-
"output": output,
|
|
90
|
-
"instance_type": instance_type,
|
|
91
|
-
"description": description,
|
|
92
|
-
**kwargs,
|
|
93
|
-
}
|
|
94
|
-
)
|
|
95
|
-
|
|
96
|
-
request_info = self.client.get_json_dict(self._get_processing_url(), post_values=payload, use_session=True)
|
|
97
|
-
|
|
98
|
-
return BatchProcessRequest.from_dict(request_info)
|
|
99
|
-
|
|
100
|
-
@staticmethod
|
|
101
|
-
def geopackage_input(geopackage_specification: AccessSpecification) -> JsonDict:
|
|
102
|
-
"""A helper method to build a suitable dictionary for the `input` field.
|
|
103
|
-
|
|
104
|
-
:param geopackage_specification: A specification of the S3 path for the Geopackage. Can be built using the
|
|
105
|
-
`s3_specification` helper method.
|
|
106
|
-
"""
|
|
107
|
-
return {"type": "geopackage", "features": geopackage_specification}
|
|
108
|
-
|
|
109
|
-
@staticmethod
|
|
110
|
-
def tiling_grid_input(
|
|
111
|
-
grid_id: int, resolution: float, buffer_x: Optional[int] = None, buffer_y: Optional[int] = None, **kwargs: Any
|
|
112
|
-
) -> JsonDict:
|
|
113
|
-
"""A helper method to build a dictionary with tiling grid parameters for the `input` field.
|
|
114
|
-
|
|
115
|
-
:param grid_id: An ID of a tiling grid
|
|
116
|
-
:param resolution: A grid resolution
|
|
117
|
-
:param buffer_x: Will expand each output tile horizontally (left and right) by specified number of pixels.
|
|
118
|
-
:param buffer_y: Will expand each output tile vertically (up and down) by specified number of pixels.
|
|
119
|
-
:param kwargs: Any other arguments to be added to a dictionary of parameters
|
|
120
|
-
"""
|
|
121
|
-
return remove_undefined(
|
|
122
|
-
{
|
|
123
|
-
"type": "tiling-grid",
|
|
124
|
-
"id": grid_id,
|
|
125
|
-
"resolution": resolution,
|
|
126
|
-
"bufferX": buffer_x,
|
|
127
|
-
"bufferY": buffer_y,
|
|
128
|
-
**kwargs,
|
|
129
|
-
}
|
|
130
|
-
)
|
|
131
|
-
|
|
132
|
-
@staticmethod
|
|
133
|
-
def raster_output(
|
|
134
|
-
delivery: AccessSpecification,
|
|
135
|
-
*,
|
|
136
|
-
overwrite: Optional[bool] = None,
|
|
137
|
-
skip_existing: Optional[bool] = None,
|
|
138
|
-
cog_output: Optional[bool] = None,
|
|
139
|
-
cog_parameters: Optional[Dict[str, Any]] = None,
|
|
140
|
-
create_collection: Optional[bool] = None,
|
|
141
|
-
collection_id: Optional[str] = None,
|
|
142
|
-
**kwargs: Any,
|
|
143
|
-
) -> Dict[str, Any]:
|
|
144
|
-
"""A helper method to build a dictionary specifying raster output
|
|
145
|
-
|
|
146
|
-
:param delivery: An S3 access specification containing a path or a template on an s3 bucket where to store
|
|
147
|
-
results. You can use the `s3_specification` method for construction. For more information on templates see
|
|
148
|
-
documentation.
|
|
149
|
-
:param overwrite: A flag specifying if a request should overwrite existing outputs without failing
|
|
150
|
-
:param skip_existing: A flag specifying if existing outputs should be overwritten
|
|
151
|
-
:param cog_output: A flag specifying if outputs should be written in COGs (cloud-optimized GeoTIFFs) or
|
|
152
|
-
normal GeoTIFFs
|
|
153
|
-
:param cog_parameters: A dictionary specifying COG creation parameters. See documentation for more info.
|
|
154
|
-
:param create_collection: If True the results will be written in COGs and a batch collection will be created
|
|
155
|
-
:param collection_id: If True results will be added to an existing collection
|
|
156
|
-
:param kwargs: Any other arguments to be added to a dictionary of parameters
|
|
157
|
-
"""
|
|
158
|
-
return remove_undefined(
|
|
159
|
-
{
|
|
160
|
-
"type": "raster",
|
|
161
|
-
"delivery": delivery,
|
|
162
|
-
"overwrite": overwrite,
|
|
163
|
-
"skipExisting": skip_existing,
|
|
164
|
-
"cogOutput": cog_output,
|
|
165
|
-
"cogParameters": cog_parameters,
|
|
166
|
-
"createCollection": create_collection,
|
|
167
|
-
"collectionId": collection_id,
|
|
168
|
-
**kwargs,
|
|
169
|
-
}
|
|
170
|
-
)
|
|
171
|
-
|
|
172
|
-
@staticmethod
|
|
173
|
-
def zarr_output(
|
|
174
|
-
delivery: AccessSpecification,
|
|
175
|
-
*,
|
|
176
|
-
group: Optional[Dict[str, Any]] = None,
|
|
177
|
-
array_parameters: Optional[Dict[str, Any]] = None,
|
|
178
|
-
array_overrides: Optional[Dict[str, Any]] = None,
|
|
179
|
-
**kwargs: Any,
|
|
180
|
-
) -> JsonDict:
|
|
181
|
-
"""A helper method to build a dictionary specifying Zarr output. See documentation for more information on
|
|
182
|
-
each parameter.
|
|
183
|
-
|
|
184
|
-
`Batch Process V2 <https://docs.sentinel-hub.com/api/latest/api/batchv2/>`__
|
|
185
|
-
|
|
186
|
-
:param delivery: An S3 access specification containing a path or a template on an s3 bucket where to store
|
|
187
|
-
results. You can use the `s3_specification` method for construction. For more information on templates see
|
|
188
|
-
documentation.
|
|
189
|
-
:param group: Zarr group level parameters
|
|
190
|
-
:param array_parameters: Parameters that will be used for all output arrays, except where overriden with
|
|
191
|
-
`array_overrides`. Required unless `array_overrides` includes all required fields for all output arrays.
|
|
192
|
-
:param array_overrides: Overrides the values of `array_arameters` for individual arrays.
|
|
193
|
-
:param kwargs: Any other arguments to be added to a dictionary of parameters
|
|
194
|
-
"""
|
|
195
|
-
return remove_undefined(
|
|
196
|
-
{
|
|
197
|
-
"type": "zarr",
|
|
198
|
-
"delivery": delivery,
|
|
199
|
-
"group": group,
|
|
200
|
-
"arrayParameter": array_parameters,
|
|
201
|
-
"arrayOverrides": array_overrides,
|
|
202
|
-
**kwargs,
|
|
203
|
-
}
|
|
204
|
-
)
|
|
205
|
-
|
|
206
|
-
def iter_requests(
|
|
207
|
-
self, user_id: Optional[str] = None, search: Optional[str] = None, sort: Optional[str] = None, **kwargs: Any
|
|
208
|
-
) -> Iterator["BatchProcessRequest"]:
|
|
209
|
-
"""Iterate existing batch requests
|
|
210
|
-
|
|
211
|
-
`Batch Process V2 <https://docs.sentinel-hub.com/api/latest/api/batchv2/>`__
|
|
212
|
-
|
|
213
|
-
:param user_id: Filter requests by a user id who defined a request
|
|
214
|
-
:param search: A search query to filter requests
|
|
215
|
-
:param sort: A sort query
|
|
216
|
-
:param kwargs: Any additional parameters to include in a request query
|
|
217
|
-
:return: An iterator over existing batch requests
|
|
218
|
-
"""
|
|
219
|
-
params = remove_undefined({"userid": user_id, "search": search, "sort": sort, **kwargs})
|
|
220
|
-
feature_iterator = SentinelHubFeatureIterator(
|
|
221
|
-
client=self.client, url=self._get_processing_url(), params=params, exception_message="No requests found"
|
|
222
|
-
)
|
|
223
|
-
for request_info in feature_iterator:
|
|
224
|
-
yield BatchProcessRequest.from_dict(request_info)
|
|
225
|
-
|
|
226
|
-
def get_request(self, batch_request: BatchRequestType) -> "BatchProcessRequest":
|
|
227
|
-
"""Collects information about a single batch request.
|
|
228
|
-
|
|
229
|
-
`Batch Process V2 <https://docs.sentinel-hub.com/api/latest/api/batchv2/>`__
|
|
230
|
-
"""
|
|
231
|
-
request_id = self._parse_request_id(batch_request)
|
|
232
|
-
request_info = self.client.get_json_dict(url=self._get_processing_url(request_id), use_session=True)
|
|
233
|
-
return BatchProcessRequest.from_dict(request_info)
|
|
234
|
-
|
|
235
|
-
def update_request(self, batch_request: BatchRequestType, description: str) -> Json:
|
|
236
|
-
"""Update certain batch job request parameters. Can only update requests that are not currently being processed.
|
|
237
|
-
|
|
238
|
-
`Batch Process V2 <https://docs.sentinel-hub.com/api/latest/api/batchv2/>`__
|
|
239
|
-
|
|
240
|
-
:param batch_request: Batch request ID, a dictionary containing an "ID" field, or a BatchProcessRequest.
|
|
241
|
-
:param description: A description of a batch request to be updated.
|
|
242
|
-
"""
|
|
243
|
-
request_id = self._parse_request_id(batch_request)
|
|
244
|
-
|
|
245
|
-
return self.client.get_json(
|
|
246
|
-
url=self._get_processing_url(request_id),
|
|
247
|
-
post_values={"description": description},
|
|
248
|
-
request_type=RequestType.PUT,
|
|
249
|
-
use_session=True,
|
|
250
|
-
)
|
|
251
|
-
|
|
252
|
-
def start_analysis(self, batch_request: BatchRequestType) -> Json:
|
|
253
|
-
"""Starts analysis of a batch job request
|
|
254
|
-
|
|
255
|
-
:param batch_request: Batch request ID, a dictionary containing an "ID" field, or a BatchProcessRequest.
|
|
256
|
-
"""
|
|
257
|
-
return self._call_job(batch_request, "analyse")
|
|
258
|
-
|
|
259
|
-
def start_job(self, batch_request: BatchRequestType) -> Json:
|
|
260
|
-
"""Starts running a batch job
|
|
261
|
-
|
|
262
|
-
:param batch_request: Batch request ID, a dictionary containing an "ID" field, or a BatchProcessRequest.
|
|
263
|
-
"""
|
|
264
|
-
return self._call_job(batch_request, "start")
|
|
265
|
-
|
|
266
|
-
def stop_job(self, batch_request: BatchRequestType) -> Json:
|
|
267
|
-
"""Stops a batch job
|
|
268
|
-
|
|
269
|
-
:param batch_request: Batch request ID, a dictionary containing an "ID" field, or a BatchProcessRequest.
|
|
270
|
-
"""
|
|
271
|
-
return self._call_job(batch_request, "stop")
|
|
272
|
-
|
|
273
|
-
def iter_tiling_grids(self, **kwargs: Any) -> SentinelHubFeatureIterator:
|
|
274
|
-
"""An iterator over tiling grids
|
|
275
|
-
|
|
276
|
-
`Batch Process V2 <https://docs.sentinel-hub.com/api/latest/api/batchv2/>`__
|
|
277
|
-
|
|
278
|
-
:param kwargs: Any other request query parameters
|
|
279
|
-
:return: An iterator over tiling grid definitions
|
|
280
|
-
"""
|
|
281
|
-
return SentinelHubFeatureIterator(
|
|
282
|
-
client=self.client,
|
|
283
|
-
url=f"{self.service_url}/tilinggrids",
|
|
284
|
-
params=remove_undefined(kwargs),
|
|
285
|
-
exception_message="Failed to obtain information about available tiling grids",
|
|
286
|
-
)
|
|
287
|
-
|
|
288
|
-
def get_tiling_grid(self, grid_id: int) -> JsonDict:
|
|
289
|
-
"""Provides a single tiling grid
|
|
290
|
-
|
|
291
|
-
`Batch Process V2 <https://docs.sentinel-hub.com/api/latest/api/batchv2/>`__
|
|
292
|
-
|
|
293
|
-
:param grid_id: An ID of a requested tiling grid
|
|
294
|
-
:return: A tiling grid definition
|
|
295
|
-
"""
|
|
296
|
-
url = f"{self.service_url}/tilinggrids/{grid_id}"
|
|
297
|
-
return self.client.get_json_dict(url=url, use_session=True)
|
|
298
|
-
|
|
299
|
-
|
|
300
|
-
@dataclass_json(letter_case=LetterCase.CAMEL, undefined=Undefined.INCLUDE)
|
|
301
|
-
@dataclass(repr=False)
|
|
302
|
-
class BatchProcessRequest(BaseBatchRequest): # pylint: disable=abstract-method
|
|
303
|
-
"""A dataclass object that holds information about a batch request"""
|
|
304
|
-
|
|
305
|
-
# dataclass_json doesn't handle parameter inheritance correctly
|
|
306
|
-
# pylint: disable=invalid-name
|
|
307
|
-
|
|
308
|
-
request_id: str = field(metadata=dataclass_config(field_name="id"))
|
|
309
|
-
request: dict
|
|
310
|
-
domain_account_id: str
|
|
311
|
-
status: BatchRequestStatus = field(metadata=enum_config(BatchRequestStatus))
|
|
312
|
-
error: Optional[str] = None
|
|
313
|
-
user_action: Optional[BatchUserAction] = field(metadata=enum_config(BatchUserAction), default=None)
|
|
314
|
-
user_action_updated: Optional[dt.datetime] = field(metadata=datetime_config, default=None)
|
|
315
|
-
created: Optional[dt.datetime] = field(metadata=datetime_config, default=None)
|
|
316
|
-
completion_percentage: float = 0
|
|
317
|
-
last_updated: Optional[dt.datetime] = field(metadata=datetime_config, default=None)
|
|
318
|
-
cost_PU: Optional[float] = field(metadata=dataclass_config(field_name="costPU"), default=None) # noqa: N815
|
|
319
|
-
stopped_status_reason: Optional[StoppedStatusReason] = field(
|
|
320
|
-
metadata=enum_config(StoppedStatusReason), default=None
|
|
321
|
-
)
|
|
322
|
-
|
|
323
|
-
other_data: CatchAll = field(default_factory=dict)
|
|
324
|
-
|
|
325
|
-
_REPR_PARAM_NAMES = ("request_id", "created", "status", "completion_percentage", "user_action", "cost_PU")
|
|
10
|
+
warnings.warn(
|
|
11
|
+
"The module sentinelhub.api.batch.process_v2 has been renamed, please use sentinelhub.api.batch.process instead.",
|
|
12
|
+
SHDeprecationWarning,
|
|
13
|
+
stacklevel=2,
|
|
14
|
+
)
|
sentinelhub/api/batch/utils.py
CHANGED
|
@@ -6,7 +6,6 @@ from __future__ import annotations
|
|
|
6
6
|
|
|
7
7
|
import logging
|
|
8
8
|
import time
|
|
9
|
-
from collections import defaultdict
|
|
10
9
|
from typing import Union
|
|
11
10
|
|
|
12
11
|
from tqdm.auto import tqdm
|
|
@@ -14,13 +13,11 @@ from tqdm.auto import tqdm
|
|
|
14
13
|
from ...config import SHConfig
|
|
15
14
|
from ...types import JsonDict
|
|
16
15
|
from .base import BatchRequestStatus
|
|
17
|
-
from .process import
|
|
18
|
-
from .process_v2 import BatchProcessClient, BatchProcessRequest
|
|
16
|
+
from .process import BatchProcessClient, BatchProcessRequest
|
|
19
17
|
from .statistical import BatchStatisticalRequest, SentinelHubBatchStatistical
|
|
20
18
|
|
|
21
19
|
LOGGER = logging.getLogger(__name__)
|
|
22
20
|
|
|
23
|
-
BatchProcessRequestSpec = Union[str, dict, BatchRequest]
|
|
24
21
|
BatchStatisticalRequestSpec = Union[str, dict, BatchStatisticalRequest]
|
|
25
22
|
|
|
26
23
|
|
|
@@ -32,81 +29,6 @@ _MIN_ANALYSIS_SLEEP_TIME = 5
|
|
|
32
29
|
_DEFAULT_ANALYSIS_SLEEP_TIME = 10
|
|
33
30
|
|
|
34
31
|
|
|
35
|
-
def monitor_batch_job(
|
|
36
|
-
batch_request: BatchProcessRequestSpec,
|
|
37
|
-
config: SHConfig | None = None,
|
|
38
|
-
sleep_time: int = _DEFAULT_SLEEP_TIME,
|
|
39
|
-
analysis_sleep_time: int = _DEFAULT_ANALYSIS_SLEEP_TIME,
|
|
40
|
-
) -> defaultdict[BatchTileStatus, list[dict]]:
|
|
41
|
-
"""A utility function that keeps checking for number of processed tiles until the given batch request finishes.
|
|
42
|
-
During the process it shows a progress bar and at the end it reports information about finished and failed tiles.
|
|
43
|
-
|
|
44
|
-
Notes:
|
|
45
|
-
|
|
46
|
-
- Before calling this function make sure to start a batch job by calling `SentinelHubBatch.start_job` method. In
|
|
47
|
-
case a batch job is still being analysed this function will wait until the analysis ends.
|
|
48
|
-
- This function will be continuously collecting tile information from Sentinel Hub service. To avoid making too
|
|
49
|
-
many requests please make sure to adjust `sleep_time` parameter according to the size of your job. Larger jobs
|
|
50
|
-
don't need very frequent tile status updates.
|
|
51
|
-
- Some information about the progress of this function is reported to logging level INFO.
|
|
52
|
-
|
|
53
|
-
:param batch_request: An object with information about a batch request. Alternatively, it could only be a batch
|
|
54
|
-
request id or a payload.
|
|
55
|
-
:param config: A configuration object with required parameters `sh_client_id`, `sh_client_secret`, and
|
|
56
|
-
`sh_auth_base_url` which is used for authentication and `sh_base_url` which defines the service deployment
|
|
57
|
-
where Batch API will be called.
|
|
58
|
-
:param sleep_time: Number of seconds to sleep between consecutive progress bar updates.
|
|
59
|
-
:param analysis_sleep_time: Number of seconds between consecutive status updates during analysis phase.
|
|
60
|
-
:return: A dictionary mapping a tile status to a list of tile payloads.
|
|
61
|
-
"""
|
|
62
|
-
if sleep_time < _MIN_SLEEP_TIME:
|
|
63
|
-
raise ValueError(f"To avoid making too many service requests please set sleep_time>={_MIN_SLEEP_TIME}")
|
|
64
|
-
|
|
65
|
-
batch_request = monitor_batch_analysis(batch_request, config=config, sleep_time=analysis_sleep_time)
|
|
66
|
-
if batch_request.status is BatchRequestStatus.PROCESSING:
|
|
67
|
-
LOGGER.info("Batch job is running")
|
|
68
|
-
|
|
69
|
-
batch_client = SentinelHubBatch(config=config)
|
|
70
|
-
|
|
71
|
-
tiles_per_status = _get_batch_tiles_per_status(batch_request, batch_client)
|
|
72
|
-
success_count = len(tiles_per_status[BatchTileStatus.PROCESSED])
|
|
73
|
-
finished_count = success_count + len(tiles_per_status[BatchTileStatus.FAILED])
|
|
74
|
-
|
|
75
|
-
progress_bar = tqdm(total=batch_request.tile_count, initial=finished_count, desc="Progress rate")
|
|
76
|
-
success_bar = tqdm(total=finished_count, initial=success_count, desc="Success rate")
|
|
77
|
-
|
|
78
|
-
monitoring_status = [BatchRequestStatus.ANALYSIS_DONE, BatchRequestStatus.PROCESSING]
|
|
79
|
-
with progress_bar, success_bar:
|
|
80
|
-
while finished_count < batch_request.tile_count and batch_request.status in monitoring_status:
|
|
81
|
-
time.sleep(sleep_time)
|
|
82
|
-
batch_request = batch_client.get_request(batch_request)
|
|
83
|
-
|
|
84
|
-
tiles_per_status = _get_batch_tiles_per_status(batch_request, batch_client)
|
|
85
|
-
new_success_count = len(tiles_per_status[BatchTileStatus.PROCESSED])
|
|
86
|
-
new_finished_count = new_success_count + len(tiles_per_status[BatchTileStatus.FAILED])
|
|
87
|
-
|
|
88
|
-
progress_bar.update(new_finished_count - finished_count)
|
|
89
|
-
if new_finished_count != finished_count:
|
|
90
|
-
success_bar.total = new_finished_count
|
|
91
|
-
success_bar.refresh()
|
|
92
|
-
success_bar.update(new_success_count - success_count)
|
|
93
|
-
|
|
94
|
-
finished_count = new_finished_count
|
|
95
|
-
success_count = new_success_count
|
|
96
|
-
|
|
97
|
-
failed_tiles_num = finished_count - success_count
|
|
98
|
-
if failed_tiles_num:
|
|
99
|
-
LOGGER.info("Batch job failed for %d tiles", failed_tiles_num)
|
|
100
|
-
|
|
101
|
-
while batch_request.status is BatchRequestStatus.PROCESSING:
|
|
102
|
-
LOGGER.info("Waiting on batch job status update.")
|
|
103
|
-
time.sleep(sleep_time)
|
|
104
|
-
batch_request = batch_client.get_request(batch_request)
|
|
105
|
-
|
|
106
|
-
LOGGER.info("Batch job finished with status %s", batch_request.status.value)
|
|
107
|
-
return tiles_per_status
|
|
108
|
-
|
|
109
|
-
|
|
110
32
|
def monitor_batch_process_job(
|
|
111
33
|
request: BatchProcessRequest,
|
|
112
34
|
client: BatchProcessClient,
|
|
@@ -155,23 +77,6 @@ def monitor_batch_process_job(
|
|
|
155
77
|
return batch_request
|
|
156
78
|
|
|
157
79
|
|
|
158
|
-
def _get_batch_tiles_per_status(
|
|
159
|
-
batch_request: BatchRequest, batch_client: SentinelHubBatch
|
|
160
|
-
) -> defaultdict[BatchTileStatus, list[dict]]:
|
|
161
|
-
"""A helper function that queries information about batch tiles and returns information about tiles, grouped by
|
|
162
|
-
tile status.
|
|
163
|
-
|
|
164
|
-
:return: A dictionary mapping a tile status to a list of tile payloads.
|
|
165
|
-
"""
|
|
166
|
-
tiles_per_status = defaultdict(list)
|
|
167
|
-
|
|
168
|
-
for tile in batch_client.iter_tiles(batch_request):
|
|
169
|
-
status = BatchTileStatus(tile["status"])
|
|
170
|
-
tiles_per_status[status].append(tile)
|
|
171
|
-
|
|
172
|
-
return tiles_per_status
|
|
173
|
-
|
|
174
|
-
|
|
175
80
|
def monitor_batch_statistical_job(
|
|
176
81
|
batch_request: BatchStatisticalRequestSpec,
|
|
177
82
|
config: SHConfig | None = None,
|
|
@@ -217,38 +122,6 @@ def monitor_batch_statistical_job(
|
|
|
217
122
|
return request_status
|
|
218
123
|
|
|
219
124
|
|
|
220
|
-
def monitor_batch_analysis(
|
|
221
|
-
batch_request: BatchProcessRequestSpec,
|
|
222
|
-
config: SHConfig | None = None,
|
|
223
|
-
sleep_time: int = _DEFAULT_ANALYSIS_SLEEP_TIME,
|
|
224
|
-
) -> BatchRequest:
|
|
225
|
-
"""A utility function that is waiting until analysis phase of a batch job finishes and regularly checks its status.
|
|
226
|
-
In case analysis phase failed it raises an error at the end.
|
|
227
|
-
|
|
228
|
-
:param batch_request: An object with information about a batch request. Alternatively, it could only be a batch
|
|
229
|
-
request id or a payload.
|
|
230
|
-
:param config: A configuration object with required parameters `sh_client_id`, `sh_client_secret`, and
|
|
231
|
-
`sh_auth_base_url` which is used for authentication and `sh_base_url` which defines the service deployment
|
|
232
|
-
where Batch API will be called.
|
|
233
|
-
:param sleep_time: Number of seconds between consecutive status updates during analysis phase.
|
|
234
|
-
:return: Batch request info
|
|
235
|
-
"""
|
|
236
|
-
if sleep_time < _MIN_ANALYSIS_SLEEP_TIME:
|
|
237
|
-
raise ValueError(
|
|
238
|
-
f"To avoid making too many service requests please set analysis sleep time >={_MIN_ANALYSIS_SLEEP_TIME}"
|
|
239
|
-
)
|
|
240
|
-
|
|
241
|
-
batch_client = SentinelHubBatch(config=config)
|
|
242
|
-
batch_request = batch_client.get_request(batch_request)
|
|
243
|
-
while batch_request.status in [BatchRequestStatus.CREATED, BatchRequestStatus.ANALYSING]:
|
|
244
|
-
LOGGER.info("Batch job has a status %s, sleeping for %d seconds", batch_request.status.value, sleep_time)
|
|
245
|
-
time.sleep(sleep_time)
|
|
246
|
-
batch_request = batch_client.get_request(batch_request)
|
|
247
|
-
|
|
248
|
-
batch_request.raise_for_status(status=[BatchRequestStatus.FAILED, BatchRequestStatus.CANCELED])
|
|
249
|
-
return batch_request
|
|
250
|
-
|
|
251
|
-
|
|
252
125
|
def monitor_batch_process_analysis(
|
|
253
126
|
request: BatchProcessRequest,
|
|
254
127
|
client: BatchProcessClient,
|
|
@@ -272,7 +145,7 @@ def monitor_batch_process_analysis(
|
|
|
272
145
|
time.sleep(sleep_time)
|
|
273
146
|
batch_request = client.get_request(batch_request)
|
|
274
147
|
|
|
275
|
-
batch_request.raise_for_status(status=[BatchRequestStatus.FAILED, BatchRequestStatus.
|
|
148
|
+
batch_request.raise_for_status(status=[BatchRequestStatus.FAILED, BatchRequestStatus.STOPPED])
|
|
276
149
|
return batch_request
|
|
277
150
|
|
|
278
151
|
|
|
@@ -305,5 +178,5 @@ def monitor_batch_statistical_analysis(
|
|
|
305
178
|
request_status = BatchRequestStatus(batch_client.get_status(batch_request)["status"])
|
|
306
179
|
|
|
307
180
|
batch_request = batch_client.get_request(batch_request)
|
|
308
|
-
batch_request.raise_for_status(status=[BatchRequestStatus.FAILED, BatchRequestStatus.
|
|
181
|
+
batch_request.raise_for_status(status=[BatchRequestStatus.FAILED, BatchRequestStatus.STOPPED])
|
|
309
182
|
return batch_request
|
sentinelhub/areas.py
CHANGED
|
@@ -17,13 +17,12 @@ import shapely.ops
|
|
|
17
17
|
from shapely.geometry import GeometryCollection, MultiPolygon, Polygon
|
|
18
18
|
from shapely.geometry.base import BaseGeometry
|
|
19
19
|
|
|
20
|
-
from .api import
|
|
20
|
+
from .api import SentinelHubCatalog
|
|
21
21
|
from .config import SHConfig
|
|
22
22
|
from .constants import CRS
|
|
23
23
|
from .data_collections import DataCollection
|
|
24
24
|
from .geo_utils import transform_point
|
|
25
25
|
from .geometry import BBox, Geometry, _BaseGeometry
|
|
26
|
-
from .types import JsonDict
|
|
27
26
|
|
|
28
27
|
T = TypeVar("T", float, int)
|
|
29
28
|
|
|
@@ -534,7 +533,7 @@ class BaseUtmSplitter(AreaSplitter, metaclass=ABCMeta):
|
|
|
534
533
|
geo_object for geo_object in intersection if isinstance(geo_object, (Polygon, MultiPolygon))
|
|
535
534
|
)
|
|
536
535
|
|
|
537
|
-
if
|
|
536
|
+
if intersection.area > 0:
|
|
538
537
|
intersection = Geometry(intersection, CRS.WGS84).transform(utm_crs)
|
|
539
538
|
|
|
540
539
|
bbox_partition = self._align_bbox_to_size(intersection.bbox).get_partition(size_x=size_x, size_y=size_y)
|
|
@@ -624,81 +623,6 @@ class UtmZoneSplitter(BaseUtmSplitter):
|
|
|
624
623
|
return list(zip(utm_geom_list, utm_prop_list))
|
|
625
624
|
|
|
626
625
|
|
|
627
|
-
class BatchSplitter(AreaSplitter):
|
|
628
|
-
"""A splitter that obtains split bounding boxes from Sentinel Hub Batch API"""
|
|
629
|
-
|
|
630
|
-
def __init__(
|
|
631
|
-
self,
|
|
632
|
-
*,
|
|
633
|
-
request_id: str | None = None,
|
|
634
|
-
batch_request: BatchRequest | None = None,
|
|
635
|
-
config: SHConfig | None = None,
|
|
636
|
-
):
|
|
637
|
-
"""
|
|
638
|
-
:param request_id: An ID of a batch request
|
|
639
|
-
:param batch_request: A batch request object. It is an alternative to the `request_id` parameter
|
|
640
|
-
:param config: A configuration object with credentials and information about which service deployment to
|
|
641
|
-
use.
|
|
642
|
-
"""
|
|
643
|
-
self.batch_client = SentinelHubBatch(config=config)
|
|
644
|
-
|
|
645
|
-
if batch_request is None:
|
|
646
|
-
if request_id is None:
|
|
647
|
-
raise ValueError("One of the parameters request_id and batch_request has to be given")
|
|
648
|
-
batch_request = self.batch_client.get_request(request_id)
|
|
649
|
-
|
|
650
|
-
self.batch_request = batch_request
|
|
651
|
-
self.tile_size = self._get_tile_size()
|
|
652
|
-
self.tile_buffer = self._get_tile_buffer()
|
|
653
|
-
|
|
654
|
-
batch_geometry: _BaseGeometry | None = batch_request.geometry or batch_request.bbox
|
|
655
|
-
if batch_geometry is None:
|
|
656
|
-
raise ValueError("Batch request has both `bbox` and `geometry` set to `None`, which is invalid.")
|
|
657
|
-
|
|
658
|
-
super().__init__([batch_geometry.geometry], batch_geometry.crs)
|
|
659
|
-
|
|
660
|
-
def _get_tile_size(self) -> tuple[float, float]:
|
|
661
|
-
"""Collects a tile size from the tiling grid info in units of the grid CRS."""
|
|
662
|
-
tiling_grid_id = self.batch_request.tiling_grid["id"]
|
|
663
|
-
grid_info = self.batch_client.get_tiling_grid(tiling_grid_id)
|
|
664
|
-
|
|
665
|
-
return grid_info["properties"]["tileWidth"], grid_info["properties"]["tileHeight"]
|
|
666
|
-
|
|
667
|
-
def _get_tile_buffer(self) -> tuple[float, float]:
|
|
668
|
-
"""Calculates tile buffer in units of the grid CRS."""
|
|
669
|
-
grid_info = self.batch_request.tiling_grid
|
|
670
|
-
resolution = grid_info["resolution"]
|
|
671
|
-
return grid_info.get("bufferX", 0) * resolution, grid_info.get("bufferY", 0) * resolution
|
|
672
|
-
|
|
673
|
-
def _make_split(self) -> tuple[list[BBox], list[dict[str, object]]]:
|
|
674
|
-
"""This method actually loads bounding boxes from the service and prepares the lists"""
|
|
675
|
-
tile_info_list = list(self.batch_client.iter_tiles(self.batch_request))
|
|
676
|
-
|
|
677
|
-
bbox_list = [self._reconstruct_bbox(tile_info) for tile_info in tile_info_list]
|
|
678
|
-
info_list = [
|
|
679
|
-
{key: value for key, value in tile_info.items() if key != "geometry"} for tile_info in tile_info_list
|
|
680
|
-
]
|
|
681
|
-
|
|
682
|
-
return bbox_list, info_list
|
|
683
|
-
|
|
684
|
-
def _reconstruct_bbox(self, tile_info: JsonDict) -> BBox:
|
|
685
|
-
"""Reconstructs a bounding box from tile and grid properties."""
|
|
686
|
-
tile_crs = CRS(tile_info["origin"]["crs"]["properties"]["name"])
|
|
687
|
-
|
|
688
|
-
upper_left_corner = tile_info["origin"]["coordinates"]
|
|
689
|
-
width, height = self.tile_size
|
|
690
|
-
|
|
691
|
-
return BBox(
|
|
692
|
-
(
|
|
693
|
-
upper_left_corner[0] - self.tile_buffer[0],
|
|
694
|
-
upper_left_corner[1] - height - self.tile_buffer[1],
|
|
695
|
-
upper_left_corner[0] + width + self.tile_buffer[0],
|
|
696
|
-
upper_left_corner[1] + self.tile_buffer[1],
|
|
697
|
-
),
|
|
698
|
-
tile_crs,
|
|
699
|
-
)
|
|
700
|
-
|
|
701
|
-
|
|
702
626
|
def _parse_to_pair(parameter: T | tuple[T, T], allowed_types: tuple[type, ...], param_name: str = "") -> tuple[T, T]:
|
|
703
627
|
"""Parses the parameters defining the splitting of the BBox."""
|
|
704
628
|
|
sentinelhub/config.py
CHANGED
|
@@ -216,5 +216,5 @@ class SHConfig(_SHConfig):
|
|
|
216
216
|
@classmethod
|
|
217
217
|
def get_config_location(cls) -> str:
|
|
218
218
|
"""Returns the default location of the user configuration file on disk."""
|
|
219
|
-
|
|
220
|
-
return os.path.join(
|
|
219
|
+
config_folder = os.getenv("XDG_CONFIG_HOME", os.path.expanduser("~/.config"))
|
|
220
|
+
return os.path.join(config_folder, "sentinelhub", "config.toml")
|
|
@@ -1,6 +1,6 @@
|
|
|
1
|
-
Metadata-Version: 2.
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
2
|
Name: sentinelhub
|
|
3
|
-
Version: 3.11.
|
|
3
|
+
Version: 3.11.2
|
|
4
4
|
Summary: Python API for Sentinel Hub
|
|
5
5
|
Project-URL: Homepage, https://github.com/sentinel-hub/sentinelhub-py
|
|
6
6
|
Project-URL: Documentation, https://sentinelhub-py.readthedocs.io
|
|
@@ -50,7 +50,7 @@ Requires-Python: >=3.8
|
|
|
50
50
|
Requires-Dist: aenum>=2.1.4
|
|
51
51
|
Requires-Dist: click
|
|
52
52
|
Requires-Dist: dataclasses-json
|
|
53
|
-
Requires-Dist: numpy
|
|
53
|
+
Requires-Dist: numpy
|
|
54
54
|
Requires-Dist: oauthlib
|
|
55
55
|
Requires-Dist: pillow>=9.2.0
|
|
56
56
|
Requires-Dist: pyproj>=2.2.0
|
|
@@ -68,7 +68,6 @@ Provides-Extra: aws
|
|
|
68
68
|
Requires-Dist: boto3; extra == 'aws'
|
|
69
69
|
Requires-Dist: botocore; extra == 'aws'
|
|
70
70
|
Provides-Extra: dev
|
|
71
|
-
Requires-Dist: basemap; extra == 'dev'
|
|
72
71
|
Requires-Dist: boto3-stubs>=1.20.0; extra == 'dev'
|
|
73
72
|
Requires-Dist: build; extra == 'dev'
|
|
74
73
|
Requires-Dist: click>=8.0.0; extra == 'dev'
|
|
@@ -160,7 +159,6 @@ A high-level overview of the main functionalities:
|
|
|
160
159
|
* [BYOC API](https://docs.sentinel-hub.com/api/latest/api/byoc/),
|
|
161
160
|
* [Statistical API](https://docs.sentinel-hub.com/api/latest/api/statistical/),
|
|
162
161
|
* [OGC services (WMS/WCS/WFS)](https://docs.sentinel-hub.com/api/latest/api/ogc/),
|
|
163
|
-
* [FIS](https://www.sentinel-hub.com/develop/api/ogc/fis-request/),
|
|
164
162
|
* authentication and rate-limit handling,
|
|
165
163
|
|
|
166
164
|
- geospatial utilities
|