azure-storage-blob 12.21.0__py3-none-any.whl → 12.22.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- azure/storage/blob/__init__.py +19 -18
- azure/storage/blob/_blob_client.py +417 -1507
- azure/storage/blob/_blob_client_helpers.py +1242 -0
- azure/storage/blob/_blob_service_client.py +82 -101
- azure/storage/blob/_blob_service_client_helpers.py +27 -0
- azure/storage/blob/_container_client.py +147 -356
- azure/storage/blob/_container_client_helpers.py +261 -0
- azure/storage/blob/_deserialize.py +68 -44
- azure/storage/blob/_download.py +114 -90
- azure/storage/blob/_encryption.py +14 -7
- azure/storage/blob/_lease.py +47 -58
- azure/storage/blob/_list_blobs_helper.py +129 -135
- azure/storage/blob/_models.py +479 -276
- azure/storage/blob/_quick_query_helper.py +30 -31
- azure/storage/blob/_serialize.py +38 -56
- azure/storage/blob/_shared/avro/datafile.py +1 -1
- azure/storage/blob/_shared/avro/datafile_async.py +1 -1
- azure/storage/blob/_shared/base_client.py +1 -1
- azure/storage/blob/_shared/base_client_async.py +1 -1
- azure/storage/blob/_shared/policies.py +8 -6
- azure/storage/blob/_shared/policies_async.py +3 -1
- azure/storage/blob/_shared/response_handlers.py +6 -2
- azure/storage/blob/_shared/shared_access_signature.py +2 -2
- azure/storage/blob/_shared/uploads.py +1 -1
- azure/storage/blob/_shared/uploads_async.py +1 -1
- azure/storage/blob/_shared_access_signature.py +70 -53
- azure/storage/blob/_upload_helpers.py +75 -68
- azure/storage/blob/_version.py +1 -1
- azure/storage/blob/aio/__init__.py +19 -11
- azure/storage/blob/aio/_blob_client_async.py +505 -255
- azure/storage/blob/aio/_blob_service_client_async.py +138 -87
- azure/storage/blob/aio/_container_client_async.py +260 -120
- azure/storage/blob/aio/_download_async.py +104 -87
- azure/storage/blob/aio/_lease_async.py +56 -55
- azure/storage/blob/aio/_list_blobs_helper.py +94 -96
- azure/storage/blob/aio/_models.py +60 -38
- azure/storage/blob/aio/_upload_helpers.py +75 -66
- {azure_storage_blob-12.21.0.dist-info → azure_storage_blob-12.22.0.dist-info}/METADATA +1 -1
- {azure_storage_blob-12.21.0.dist-info → azure_storage_blob-12.22.0.dist-info}/RECORD +42 -39
- {azure_storage_blob-12.21.0.dist-info → azure_storage_blob-12.22.0.dist-info}/LICENSE +0 -0
- {azure_storage_blob-12.21.0.dist-info → azure_storage_blob-12.22.0.dist-info}/WHEEL +0 -0
- {azure_storage_blob-12.21.0.dist-info → azure_storage_blob-12.22.0.dist-info}/top_level.txt +0 -0
@@ -4,20 +4,25 @@
|
|
4
4
|
# license information.
|
5
5
|
# --------------------------------------------------------------------------
|
6
6
|
# pylint: disable=invalid-overridden-method
|
7
|
+
# mypy: disable-error-code=override
|
7
8
|
|
9
|
+
import asyncio
|
8
10
|
import codecs
|
9
11
|
import sys
|
10
12
|
import warnings
|
11
13
|
from io import BytesIO, StringIO
|
12
14
|
from itertools import islice
|
13
|
-
from typing import
|
14
|
-
|
15
|
-
|
15
|
+
from typing import (
|
16
|
+
Any, AsyncIterator, Awaitable,
|
17
|
+
Generator, Callable, cast, Dict,
|
18
|
+
Generic, IO, Optional, overload,
|
19
|
+
Tuple, TypeVar, Union, TYPE_CHECKING
|
20
|
+
)
|
16
21
|
|
17
22
|
from azure.core.exceptions import HttpResponseError
|
18
23
|
|
19
24
|
from .._shared.request_handlers import validate_and_format_range_headers
|
20
|
-
from .._shared.response_handlers import
|
25
|
+
from .._shared.response_handlers import parse_length_from_content_range, process_storage_error
|
21
26
|
from .._deserialize import deserialize_blob_properties, get_page_ranges_result
|
22
27
|
from .._download import process_range_and_offset, _ChunkDownloader
|
23
28
|
from .._encryption import (
|
@@ -27,17 +32,25 @@ from .._encryption import (
|
|
27
32
|
parse_encryption_data
|
28
33
|
)
|
29
34
|
|
35
|
+
if TYPE_CHECKING:
|
36
|
+
from codecs import IncrementalDecoder
|
37
|
+
from .._encryption import _EncryptionData
|
38
|
+
from .._generated.aio import AzureBlobStorage
|
39
|
+
from .._models import BlobProperties
|
40
|
+
from .._shared.models import StorageConfiguration
|
41
|
+
|
42
|
+
|
30
43
|
T = TypeVar('T', bytes, str)
|
31
44
|
|
32
45
|
|
33
|
-
async def process_content(data, start_offset, end_offset, encryption):
|
46
|
+
async def process_content(data: Any, start_offset: int, end_offset: int, encryption: Dict[str, Any]) -> bytes:
|
34
47
|
if data is None:
|
35
48
|
raise ValueError("Response cannot be None.")
|
36
|
-
content = data.response.body()
|
49
|
+
content = cast(bytes, data.response.body())
|
37
50
|
if encryption.get('key') is not None or encryption.get('resolver') is not None:
|
38
51
|
try:
|
39
52
|
return decrypt_blob(
|
40
|
-
encryption.get('required'),
|
53
|
+
encryption.get('required') or False,
|
41
54
|
encryption.get('key'),
|
42
55
|
encryption.get('resolver'),
|
43
56
|
content,
|
@@ -53,12 +66,12 @@ async def process_content(data, start_offset, end_offset, encryption):
|
|
53
66
|
|
54
67
|
|
55
68
|
class _AsyncChunkDownloader(_ChunkDownloader):
|
56
|
-
def __init__(self, **kwargs):
|
69
|
+
def __init__(self, **kwargs: Any) -> None:
|
57
70
|
super(_AsyncChunkDownloader, self).__init__(**kwargs)
|
58
|
-
self.
|
59
|
-
self.
|
71
|
+
self.stream_lock_async = asyncio.Lock() if kwargs.get('parallel') else None
|
72
|
+
self.progress_lock_async = asyncio.Lock() if kwargs.get('parallel') else None
|
60
73
|
|
61
|
-
async def process_chunk(self, chunk_start):
|
74
|
+
async def process_chunk(self, chunk_start: int) -> None:
|
62
75
|
chunk_start, chunk_end = self._calculate_range(chunk_start)
|
63
76
|
chunk_data, _ = await self._download_chunk(chunk_start, chunk_end - 1)
|
64
77
|
length = chunk_end - chunk_start
|
@@ -66,29 +79,32 @@ class _AsyncChunkDownloader(_ChunkDownloader):
|
|
66
79
|
await self._write_to_stream(chunk_data, chunk_start)
|
67
80
|
await self._update_progress(length)
|
68
81
|
|
69
|
-
async def yield_chunk(self, chunk_start):
|
82
|
+
async def yield_chunk(self, chunk_start: int) -> Tuple[bytes, int]:
|
70
83
|
chunk_start, chunk_end = self._calculate_range(chunk_start)
|
71
84
|
return await self._download_chunk(chunk_start, chunk_end - 1)
|
72
85
|
|
73
|
-
async def _update_progress(self, length):
|
74
|
-
if self.
|
75
|
-
async with self.
|
86
|
+
async def _update_progress(self, length: int) -> None:
|
87
|
+
if self.progress_lock_async:
|
88
|
+
async with self.progress_lock_async:
|
76
89
|
self.progress_total += length
|
77
90
|
else:
|
78
91
|
self.progress_total += length
|
79
92
|
|
80
93
|
if self.progress_hook:
|
81
|
-
await
|
94
|
+
await cast(Callable[[int, Optional[int]], Awaitable[Any]], self.progress_hook)(
|
95
|
+
self.progress_total, self.total_size)
|
82
96
|
|
83
|
-
async def _write_to_stream(self, chunk_data, chunk_start):
|
84
|
-
if self.
|
85
|
-
async with self.
|
97
|
+
async def _write_to_stream(self, chunk_data: bytes, chunk_start: int) -> None:
|
98
|
+
if self.stream_lock_async:
|
99
|
+
async with self.stream_lock_async:
|
86
100
|
self.stream.seek(self.stream_start + (chunk_start - self.start_index))
|
87
101
|
self.stream.write(chunk_data)
|
88
102
|
else:
|
89
103
|
self.stream.write(chunk_data)
|
90
104
|
|
91
|
-
async def _download_chunk(self, chunk_start, chunk_end):
|
105
|
+
async def _download_chunk(self, chunk_start: int, chunk_end: int) -> Tuple[bytes, int]:
|
106
|
+
if self.encryption_options is None:
|
107
|
+
raise ValueError("Required argument is missing: encryption_options")
|
92
108
|
download_range, offset = process_range_and_offset(
|
93
109
|
chunk_start, chunk_end, chunk_end, self.encryption_options, self.encryption_data
|
94
110
|
)
|
@@ -105,14 +121,14 @@ class _AsyncChunkDownloader(_ChunkDownloader):
|
|
105
121
|
check_content_md5=self.validate_content
|
106
122
|
)
|
107
123
|
try:
|
108
|
-
_, response = await self.client.download(
|
124
|
+
_, response = await cast(Awaitable[Any], self.client.download(
|
109
125
|
range=range_header,
|
110
126
|
range_get_content_md5=range_validation,
|
111
127
|
validate_content=self.validate_content,
|
112
128
|
data_stream_total=self.total_size,
|
113
129
|
download_stream_current=self.progress_total,
|
114
130
|
**self.request_options
|
115
|
-
)
|
131
|
+
))
|
116
132
|
|
117
133
|
except HttpResponseError as error:
|
118
134
|
process_storage_error(error)
|
@@ -131,25 +147,25 @@ class _AsyncChunkDownloader(_ChunkDownloader):
|
|
131
147
|
class _AsyncChunkIterator(object):
|
132
148
|
"""Async iterator for chunks in blob download stream."""
|
133
149
|
|
134
|
-
def __init__(self, size, content, downloader, chunk_size):
|
150
|
+
def __init__(self, size: int, content: bytes, downloader: Optional[_AsyncChunkDownloader], chunk_size: int) -> None:
|
135
151
|
self.size = size
|
136
152
|
self._chunk_size = chunk_size
|
137
153
|
self._current_content = content
|
138
154
|
self._iter_downloader = downloader
|
139
|
-
self._iter_chunks = None
|
155
|
+
self._iter_chunks: Optional[Generator[int, None, None]] = None
|
140
156
|
self._complete = size == 0
|
141
157
|
|
142
|
-
def __len__(self):
|
158
|
+
def __len__(self) -> int:
|
143
159
|
return self.size
|
144
160
|
|
145
|
-
def __iter__(self):
|
161
|
+
def __iter__(self) -> None:
|
146
162
|
raise TypeError("Async stream must be iterated asynchronously.")
|
147
163
|
|
148
|
-
def __aiter__(self):
|
164
|
+
def __aiter__(self) -> AsyncIterator[bytes]:
|
149
165
|
return self
|
150
166
|
|
151
167
|
# Iterate through responses.
|
152
|
-
async def __anext__(self):
|
168
|
+
async def __anext__(self) -> bytes:
|
153
169
|
if self._complete:
|
154
170
|
raise StopAsyncIteration("Download complete")
|
155
171
|
if not self._iter_downloader:
|
@@ -178,46 +194,46 @@ class _AsyncChunkIterator(object):
|
|
178
194
|
|
179
195
|
return self._get_chunk_data()
|
180
196
|
|
181
|
-
def _get_chunk_data(self):
|
197
|
+
def _get_chunk_data(self) -> bytes:
|
182
198
|
chunk_data = self._current_content[: self._chunk_size]
|
183
199
|
self._current_content = self._current_content[self._chunk_size:]
|
184
200
|
return chunk_data
|
185
201
|
|
186
202
|
|
187
203
|
class StorageStreamDownloader(Generic[T]): # pylint: disable=too-many-instance-attributes
|
188
|
-
"""
|
189
|
-
|
190
|
-
:ivar str name:
|
191
|
-
The name of the blob being downloaded.
|
192
|
-
:ivar str container:
|
193
|
-
The name of the container where the blob is.
|
194
|
-
:ivar ~azure.storage.blob.BlobProperties properties:
|
195
|
-
The properties of the blob being downloaded. If only a range of the data is being
|
196
|
-
downloaded, this will be reflected in the properties.
|
197
|
-
:ivar int size:
|
198
|
-
The size of the total data in the stream. This will be the byte range if specified,
|
199
|
-
otherwise the total size of the blob.
|
204
|
+
"""
|
205
|
+
A streaming object to download from Azure Storage.
|
200
206
|
"""
|
201
207
|
|
208
|
+
name: str
|
209
|
+
"""The name of the blob being downloaded."""
|
210
|
+
container: str
|
211
|
+
"""The name of the container where the blob is."""
|
212
|
+
properties: "BlobProperties"
|
213
|
+
"""The properties of the blob being downloaded. If only a range of the data is being
|
214
|
+
downloaded, this will be reflected in the properties."""
|
215
|
+
size: int
|
216
|
+
"""The size of the total data in the stream. This will be the byte range if specified,
|
217
|
+
otherwise the total size of the blob."""
|
218
|
+
|
202
219
|
def __init__(
|
203
220
|
self,
|
204
|
-
clients=None,
|
205
|
-
config=None,
|
206
|
-
start_range=None,
|
207
|
-
end_range=None,
|
208
|
-
validate_content=None,
|
209
|
-
encryption_options=None,
|
210
|
-
max_concurrency=1,
|
211
|
-
name=None,
|
212
|
-
container=None,
|
213
|
-
encoding=None,
|
214
|
-
download_cls=None,
|
215
|
-
**kwargs
|
216
|
-
):
|
221
|
+
clients: "AzureBlobStorage" = None, # type: ignore [assignment]
|
222
|
+
config: "StorageConfiguration" = None, # type: ignore [assignment]
|
223
|
+
start_range: Optional[int] = None,
|
224
|
+
end_range: Optional[int] = None,
|
225
|
+
validate_content: bool = None, # type: ignore [assignment]
|
226
|
+
encryption_options: Dict[str, Any] = None, # type: ignore [assignment]
|
227
|
+
max_concurrency: int = 1,
|
228
|
+
name: str = None, # type: ignore [assignment]
|
229
|
+
container: str = None, # type: ignore [assignment]
|
230
|
+
encoding: Optional[str] = None,
|
231
|
+
download_cls: Optional[Callable] = None,
|
232
|
+
**kwargs: Any
|
233
|
+
) -> None:
|
217
234
|
self.name = name
|
218
235
|
self.container = container
|
219
|
-
self.
|
220
|
-
self.size = None
|
236
|
+
self.size = 0
|
221
237
|
|
222
238
|
self._clients = clients
|
223
239
|
self._config = config
|
@@ -231,10 +247,10 @@ class StorageStreamDownloader(Generic[T]): # pylint: disable=too-many-instance-
|
|
231
247
|
self._request_options = kwargs
|
232
248
|
self._response = None
|
233
249
|
self._location_mode = None
|
234
|
-
self._current_content = b''
|
250
|
+
self._current_content: Union[str, bytes] = b''
|
235
251
|
self._file_size = 0
|
236
252
|
self._non_empty_ranges = None
|
237
|
-
self._encryption_data = None
|
253
|
+
self._encryption_data: Optional["_EncryptionData"] = None
|
238
254
|
|
239
255
|
# The content download offset, after any processing (decryption), in bytes
|
240
256
|
self._download_offset = 0
|
@@ -246,7 +262,7 @@ class StorageStreamDownloader(Generic[T]): # pylint: disable=too-many-instance-
|
|
246
262
|
self._current_content_offset = 0
|
247
263
|
|
248
264
|
self._text_mode: Optional[bool] = None
|
249
|
-
self._decoder = None
|
265
|
+
self._decoder: Optional["IncrementalDecoder"] = None
|
250
266
|
# Whether the current content is the first chunk of download content or not
|
251
267
|
self._first_chunk = True
|
252
268
|
self._download_start = self._start_range or 0
|
@@ -258,13 +274,13 @@ class StorageStreamDownloader(Generic[T]): # pylint: disable=too-many-instance-
|
|
258
274
|
def __len__(self):
|
259
275
|
return self.size
|
260
276
|
|
261
|
-
async def _get_encryption_data_request(self):
|
277
|
+
async def _get_encryption_data_request(self) -> None:
|
262
278
|
# Save current request cls
|
263
279
|
download_cls = self._request_options.pop('cls', None)
|
264
280
|
# Adjust cls for get_properties
|
265
281
|
self._request_options['cls'] = deserialize_blob_properties
|
266
282
|
|
267
|
-
properties = await self._clients.blob.get_properties(**self._request_options)
|
283
|
+
properties = cast("BlobProperties", await self._clients.blob.get_properties(**self._request_options))
|
268
284
|
# This will return None if there is no encryption metadata or there are parsing errors.
|
269
285
|
# That is acceptable here, the proper error will be caught and surfaced when attempting
|
270
286
|
# to decrypt the blob.
|
@@ -273,7 +289,7 @@ class StorageStreamDownloader(Generic[T]): # pylint: disable=too-many-instance-
|
|
273
289
|
# Restore cls for download
|
274
290
|
self._request_options['cls'] = download_cls
|
275
291
|
|
276
|
-
async def _setup(self):
|
292
|
+
async def _setup(self) -> None:
|
277
293
|
if self._encryption_options.get("key") is not None or self._encryption_options.get("resolver") is not None:
|
278
294
|
await self._get_encryption_data_request()
|
279
295
|
|
@@ -299,8 +315,7 @@ class StorageStreamDownloader(Generic[T]): # pylint: disable=too-many-instance-
|
|
299
315
|
)
|
300
316
|
|
301
317
|
self._response = await self._initial_request()
|
302
|
-
|
303
|
-
self.properties = self._response.properties
|
318
|
+
self.properties = cast("BlobProperties", self._response.properties) # type: ignore [attr-defined]
|
304
319
|
self.properties.name = self.name
|
305
320
|
self.properties.container = self.container
|
306
321
|
|
@@ -313,7 +328,7 @@ class StorageStreamDownloader(Generic[T]): # pylint: disable=too-many-instance-
|
|
313
328
|
# Overwrite the content MD5 as it is the MD5 for the last range instead
|
314
329
|
# of the stored MD5
|
315
330
|
# TODO: Set to the stored MD5 when the service returns this
|
316
|
-
self.properties.content_md5 = None
|
331
|
+
self.properties.content_md5 = None # type: ignore [attr-defined]
|
317
332
|
|
318
333
|
@property
|
319
334
|
def _download_complete(self):
|
@@ -330,13 +345,13 @@ class StorageStreamDownloader(Generic[T]): # pylint: disable=too-many-instance-
|
|
330
345
|
check_content_md5=self._validate_content)
|
331
346
|
|
332
347
|
try:
|
333
|
-
location_mode, response = await self._clients.blob.download(
|
348
|
+
location_mode, response = cast(Tuple[Optional[str], Any], await self._clients.blob.download(
|
334
349
|
range=range_header,
|
335
350
|
range_get_content_md5=range_validation,
|
336
351
|
validate_content=self._validate_content,
|
337
352
|
data_stream_total=None,
|
338
353
|
download_stream_current=0,
|
339
|
-
**self._request_options)
|
354
|
+
**self._request_options))
|
340
355
|
|
341
356
|
# Check the location we read from to ensure we use the same one
|
342
357
|
# for subsequent requests.
|
@@ -350,7 +365,7 @@ class StorageStreamDownloader(Generic[T]): # pylint: disable=too-many-instance-
|
|
350
365
|
# Remove any extra encryption data size from blob size
|
351
366
|
self._file_size = adjust_blob_size_for_encryption(self._file_size, self._encryption_data)
|
352
367
|
|
353
|
-
if self._end_range is not None:
|
368
|
+
if self._end_range is not None and self._start_range is not None:
|
354
369
|
# Use the length unless it is over the end of the file
|
355
370
|
self.size = min(self._file_size - self._start_range, self._end_range - self._start_range + 1)
|
356
371
|
elif self._start_range is not None:
|
@@ -364,11 +379,11 @@ class StorageStreamDownloader(Generic[T]): # pylint: disable=too-many-instance-
|
|
364
379
|
# request a range, do a regular get request in order to get
|
365
380
|
# any properties.
|
366
381
|
try:
|
367
|
-
_, response = await self._clients.blob.download(
|
382
|
+
_, response = cast(Tuple[Optional[Any], Any], await self._clients.blob.download(
|
368
383
|
validate_content=self._validate_content,
|
369
384
|
data_stream_total=0,
|
370
385
|
download_stream_current=0,
|
371
|
-
**self._request_options)
|
386
|
+
**self._request_options))
|
372
387
|
except HttpResponseError as e:
|
373
388
|
process_storage_error(e)
|
374
389
|
|
@@ -403,8 +418,7 @@ class StorageStreamDownloader(Generic[T]): # pylint: disable=too-many-instance-
|
|
403
418
|
|
404
419
|
return response
|
405
420
|
|
406
|
-
def chunks(self):
|
407
|
-
# type: () -> AsyncIterator[bytes]
|
421
|
+
def chunks(self) -> AsyncIterator[bytes]:
|
408
422
|
"""
|
409
423
|
Iterate over chunks in the download stream. Note, the iterator returned will
|
410
424
|
iterate over the entire download content, regardless of any data that was
|
@@ -459,7 +473,7 @@ class StorageStreamDownloader(Generic[T]): # pylint: disable=too-many-instance-
|
|
459
473
|
initial_content = self._current_content if self._first_chunk else b''
|
460
474
|
return _AsyncChunkIterator(
|
461
475
|
size=self.size,
|
462
|
-
content=initial_content,
|
476
|
+
content=cast(bytes, initial_content),
|
463
477
|
downloader=iter_downloader,
|
464
478
|
chunk_size=self._config.max_chunk_get_size)
|
465
479
|
|
@@ -508,12 +522,13 @@ class StorageStreamDownloader(Generic[T]): # pylint: disable=too-many-instance-
|
|
508
522
|
# Empty blob or already read to the end
|
509
523
|
if (size == 0 or chars == 0 or
|
510
524
|
(self._download_complete and self._current_content_offset >= len(self._current_content))):
|
511
|
-
return b'' if not self._encoding else ''
|
525
|
+
return b'' if not self._encoding else '' # type: ignore [return-value]
|
512
526
|
|
513
|
-
if not self._text_mode and chars is not None:
|
527
|
+
if not self._text_mode and chars is not None and self._encoding is not None:
|
514
528
|
self._text_mode = True
|
515
529
|
self._decoder = codecs.getincrementaldecoder(self._encoding)('strict')
|
516
|
-
self._current_content = self._decoder.decode(
|
530
|
+
self._current_content = self._decoder.decode(
|
531
|
+
cast(bytes, self._current_content), final=self._download_complete)
|
517
532
|
elif self._text_mode is None:
|
518
533
|
self._text_mode = False
|
519
534
|
|
@@ -530,7 +545,7 @@ class StorageStreamDownloader(Generic[T]): # pylint: disable=too-many-instance-
|
|
530
545
|
# Start by reading from current_content
|
531
546
|
start = self._current_content_offset
|
532
547
|
length = min(len(self._current_content) - self._current_content_offset, size - count)
|
533
|
-
read = output_stream.write(self._current_content[start:start + length])
|
548
|
+
read = output_stream.write(self._current_content[start:start + length]) # type: ignore [arg-type]
|
534
549
|
|
535
550
|
count += read
|
536
551
|
self._current_content_offset += read
|
@@ -568,7 +583,7 @@ class StorageStreamDownloader(Generic[T]): # pylint: disable=too-many-instance-
|
|
568
583
|
# the requested size is reached.
|
569
584
|
chunks_iter = downloader.get_chunk_offsets()
|
570
585
|
if readall and not self._text_mode:
|
571
|
-
running_futures = [
|
586
|
+
running_futures: Any = [
|
572
587
|
asyncio.ensure_future(downloader.process_chunk(d))
|
573
588
|
for d in islice(chunks_iter, 0, self._max_concurrency)
|
574
589
|
]
|
@@ -604,13 +619,15 @@ class StorageStreamDownloader(Generic[T]): # pylint: disable=too-many-instance-
|
|
604
619
|
chunk_data, content_length = await downloader.yield_chunk(chunk)
|
605
620
|
self._download_offset += len(chunk_data)
|
606
621
|
self._raw_download_offset += content_length
|
607
|
-
self.
|
608
|
-
chunk_data, final=self._download_complete)
|
622
|
+
if self._text_mode and self._decoder is not None:
|
623
|
+
self._current_content = self._decoder.decode(chunk_data, final=self._download_complete)
|
624
|
+
else:
|
625
|
+
self._current_content = chunk_data
|
609
626
|
|
610
627
|
if remaining < len(self._current_content):
|
611
|
-
read = output_stream.write(self._current_content[:remaining])
|
628
|
+
read = output_stream.write(self._current_content[:remaining]) # type: ignore [arg-type]
|
612
629
|
else:
|
613
|
-
read = output_stream.write(self._current_content)
|
630
|
+
read = output_stream.write(self._current_content) # type: ignore [arg-type]
|
614
631
|
|
615
632
|
self._current_content_offset = read
|
616
633
|
self._read_offset += read
|
@@ -621,7 +638,7 @@ class StorageStreamDownloader(Generic[T]): # pylint: disable=too-many-instance-
|
|
621
638
|
if not self._text_mode and self._encoding:
|
622
639
|
try:
|
623
640
|
# This is technically incorrect to do, but we have it for backwards compatibility.
|
624
|
-
data = data.decode(self._encoding)
|
641
|
+
data = cast(bytes, data).decode(self._encoding)
|
625
642
|
except UnicodeDecodeError:
|
626
643
|
warnings.warn(
|
627
644
|
"Encountered a decoding error while decoding blob data from a partial read. "
|
@@ -629,7 +646,7 @@ class StorageStreamDownloader(Generic[T]): # pylint: disable=too-many-instance-
|
|
629
646
|
)
|
630
647
|
raise
|
631
648
|
|
632
|
-
return data
|
649
|
+
return data # type: ignore [return-value]
|
633
650
|
|
634
651
|
async def readall(self) -> T:
|
635
652
|
"""
|
@@ -677,7 +694,7 @@ class StorageStreamDownloader(Generic[T]): # pylint: disable=too-many-instance-
|
|
677
694
|
# Write the current content to the user stream
|
678
695
|
current_remaining = len(self._current_content) - self._current_content_offset
|
679
696
|
start = self._current_content_offset
|
680
|
-
count = stream.write(self._current_content[start:start + current_remaining])
|
697
|
+
count = stream.write(cast(bytes, self._current_content[start:start + current_remaining]))
|
681
698
|
|
682
699
|
self._current_content_offset += count
|
683
700
|
self._read_offset += count
|
@@ -710,10 +727,10 @@ class StorageStreamDownloader(Generic[T]): # pylint: disable=too-many-instance-
|
|
710
727
|
)
|
711
728
|
|
712
729
|
dl_tasks = downloader.get_chunk_offsets()
|
713
|
-
running_futures =
|
730
|
+
running_futures = {
|
714
731
|
asyncio.ensure_future(downloader.process_chunk(d))
|
715
732
|
for d in islice(dl_tasks, 0, self._max_concurrency)
|
716
|
-
|
733
|
+
}
|
717
734
|
while running_futures:
|
718
735
|
# Wait for some download to finish before adding a new one
|
719
736
|
done, running_futures = await asyncio.wait(
|
@@ -5,49 +5,55 @@
|
|
5
5
|
# --------------------------------------------------------------------------
|
6
6
|
# pylint: disable=invalid-overridden-method, docstring-keyword-should-match-keyword-only
|
7
7
|
|
8
|
-
|
9
|
-
|
10
|
-
TypeVar, TYPE_CHECKING
|
11
|
-
)
|
8
|
+
import uuid
|
9
|
+
from typing import Any, Optional, Union, TYPE_CHECKING
|
12
10
|
|
13
11
|
from azure.core.exceptions import HttpResponseError
|
14
12
|
from azure.core.tracing.decorator_async import distributed_trace_async
|
15
13
|
|
16
|
-
from .._shared.response_handlers import
|
14
|
+
from .._shared.response_handlers import process_storage_error, return_response_headers
|
17
15
|
from .._serialize import get_modify_conditions
|
18
|
-
from .._lease import BlobLeaseClient as LeaseClientBase
|
19
16
|
|
20
17
|
if TYPE_CHECKING:
|
18
|
+
from azure.storage.blob.aio import BlobClient, ContainerClient
|
21
19
|
from datetime import datetime
|
22
|
-
from .._generated.operations import BlobOperations, ContainerOperations
|
23
|
-
BlobClient = TypeVar("BlobClient")
|
24
|
-
ContainerClient = TypeVar("ContainerClient")
|
25
20
|
|
26
21
|
|
27
|
-
class BlobLeaseClient(
|
22
|
+
class BlobLeaseClient(): # pylint: disable=client-accepts-api-version-keyword
|
28
23
|
"""Creates a new BlobLeaseClient.
|
29
24
|
|
30
25
|
This client provides lease operations on a BlobClient or ContainerClient.
|
31
|
-
|
32
|
-
:
|
33
|
-
|
34
|
-
|
35
|
-
:
|
36
|
-
The ETag of the lease currently being maintained. This will be `None` if no
|
37
|
-
lease has yet been acquired or modified.
|
38
|
-
:ivar ~datetime.datetime last_modified:
|
39
|
-
The last modified timestamp of the lease currently being maintained.
|
40
|
-
This will be `None` if no lease has yet been acquired or modified.
|
41
|
-
|
42
|
-
:param client:
|
43
|
-
The client of the blob or container to lease.
|
44
|
-
:type client: ~azure.storage.blob.aio.BlobClient or
|
45
|
-
~azure.storage.blob.aio.ContainerClient
|
46
|
-
:param str lease_id:
|
47
|
-
A string representing the lease ID of an existing lease. This value does not
|
48
|
-
need to be specified in order to acquire a new lease, or break one.
|
26
|
+
:param client: The client of the blob or container to lease.
|
27
|
+
:type client: Union[BlobClient, ContainerClient]
|
28
|
+
:param lease_id: A string representing the lease ID of an existing lease. This value does not need to be
|
29
|
+
specified in order to acquire a new lease, or break one.
|
30
|
+
:type lease_id: Optional[str]
|
49
31
|
"""
|
50
32
|
|
33
|
+
id: str
|
34
|
+
"""The ID of the lease currently being maintained. This will be `None` if no
|
35
|
+
lease has yet been acquired."""
|
36
|
+
etag: Optional[str]
|
37
|
+
"""The ETag of the lease currently being maintained. This will be `None` if no
|
38
|
+
lease has yet been acquired or modified."""
|
39
|
+
last_modified: Optional["datetime"]
|
40
|
+
"""The last modified timestamp of the lease currently being maintained.
|
41
|
+
This will be `None` if no lease has yet been acquired or modified."""
|
42
|
+
|
43
|
+
def __init__( # pylint: disable=missing-client-constructor-parameter-credential, missing-client-constructor-parameter-kwargs
|
44
|
+
self, client: Union["BlobClient", "ContainerClient"],
|
45
|
+
lease_id: Optional[str] = None
|
46
|
+
) -> None:
|
47
|
+
self.id = lease_id or str(uuid.uuid4())
|
48
|
+
self.last_modified = None
|
49
|
+
self.etag = None
|
50
|
+
if hasattr(client, 'blob_name'):
|
51
|
+
self._client = client._client.blob
|
52
|
+
elif hasattr(client, 'container_name'):
|
53
|
+
self._client = client._client.container
|
54
|
+
else:
|
55
|
+
raise TypeError("Lease must use either BlobClient or ContainerClient.")
|
56
|
+
|
51
57
|
def __enter__(self):
|
52
58
|
raise TypeError("Async lease must use 'async with'.")
|
53
59
|
|
@@ -61,8 +67,7 @@ class BlobLeaseClient(LeaseClientBase):
|
|
61
67
|
await self.release()
|
62
68
|
|
63
69
|
@distributed_trace_async
|
64
|
-
async def acquire(self, lease_duration
|
65
|
-
# type: (int, Any) -> None
|
70
|
+
async def acquire(self, lease_duration: int = -1, **kwargs: Any) -> None:
|
66
71
|
"""Requests a new lease.
|
67
72
|
|
68
73
|
If the container does not have an active lease, the Blob service creates a
|
@@ -106,7 +111,7 @@ class BlobLeaseClient(LeaseClientBase):
|
|
106
111
|
"""
|
107
112
|
mod_conditions = get_modify_conditions(kwargs)
|
108
113
|
try:
|
109
|
-
response = await self._client.acquire_lease(
|
114
|
+
response: Any = await self._client.acquire_lease(
|
110
115
|
timeout=kwargs.pop('timeout', None),
|
111
116
|
duration=lease_duration,
|
112
117
|
proposed_lease_id=self.id,
|
@@ -115,13 +120,12 @@ class BlobLeaseClient(LeaseClientBase):
|
|
115
120
|
**kwargs)
|
116
121
|
except HttpResponseError as error:
|
117
122
|
process_storage_error(error)
|
118
|
-
self.id = response.get('lease_id')
|
119
|
-
self.last_modified = response.get('last_modified')
|
120
|
-
self.etag = response.get('etag')
|
123
|
+
self.id = response.get('lease_id')
|
124
|
+
self.last_modified = response.get('last_modified')
|
125
|
+
self.etag = response.get('etag')
|
121
126
|
|
122
127
|
@distributed_trace_async
|
123
|
-
async def renew(self, **kwargs):
|
124
|
-
# type: (Any) -> None
|
128
|
+
async def renew(self, **kwargs: Any) -> None:
|
125
129
|
"""Renews the lease.
|
126
130
|
|
127
131
|
The lease can be renewed if the lease ID specified in the
|
@@ -163,7 +167,7 @@ class BlobLeaseClient(LeaseClientBase):
|
|
163
167
|
"""
|
164
168
|
mod_conditions = get_modify_conditions(kwargs)
|
165
169
|
try:
|
166
|
-
response = await self._client.renew_lease(
|
170
|
+
response: Any = await self._client.renew_lease(
|
167
171
|
lease_id=self.id,
|
168
172
|
timeout=kwargs.pop('timeout', None),
|
169
173
|
modified_access_conditions=mod_conditions,
|
@@ -171,13 +175,12 @@ class BlobLeaseClient(LeaseClientBase):
|
|
171
175
|
**kwargs)
|
172
176
|
except HttpResponseError as error:
|
173
177
|
process_storage_error(error)
|
174
|
-
self.etag = response.get('etag')
|
175
|
-
self.id = response.get('lease_id')
|
176
|
-
self.last_modified = response.get('last_modified')
|
178
|
+
self.etag = response.get('etag')
|
179
|
+
self.id = response.get('lease_id')
|
180
|
+
self.last_modified = response.get('last_modified')
|
177
181
|
|
178
182
|
@distributed_trace_async
|
179
|
-
async def release(self, **kwargs):
|
180
|
-
# type: (Any) -> None
|
183
|
+
async def release(self, **kwargs: Any) -> None:
|
181
184
|
"""Release the lease.
|
182
185
|
|
183
186
|
The lease may be released if the client lease id specified matches
|
@@ -217,7 +220,7 @@ class BlobLeaseClient(LeaseClientBase):
|
|
217
220
|
"""
|
218
221
|
mod_conditions = get_modify_conditions(kwargs)
|
219
222
|
try:
|
220
|
-
response = await self._client.release_lease(
|
223
|
+
response: Any = await self._client.release_lease(
|
221
224
|
lease_id=self.id,
|
222
225
|
timeout=kwargs.pop('timeout', None),
|
223
226
|
modified_access_conditions=mod_conditions,
|
@@ -225,13 +228,12 @@ class BlobLeaseClient(LeaseClientBase):
|
|
225
228
|
**kwargs)
|
226
229
|
except HttpResponseError as error:
|
227
230
|
process_storage_error(error)
|
228
|
-
self.etag = response.get('etag')
|
229
|
-
self.id = response.get('lease_id')
|
230
|
-
self.last_modified = response.get('last_modified')
|
231
|
+
self.etag = response.get('etag')
|
232
|
+
self.id = response.get('lease_id')
|
233
|
+
self.last_modified = response.get('last_modified')
|
231
234
|
|
232
235
|
@distributed_trace_async
|
233
|
-
async def change(self, proposed_lease_id, **kwargs):
|
234
|
-
# type: (str, Any) -> None
|
236
|
+
async def change(self, proposed_lease_id: str, **kwargs: Any) -> None:
|
235
237
|
"""Change the lease ID of an active lease.
|
236
238
|
|
237
239
|
:param str proposed_lease_id:
|
@@ -270,7 +272,7 @@ class BlobLeaseClient(LeaseClientBase):
|
|
270
272
|
"""
|
271
273
|
mod_conditions = get_modify_conditions(kwargs)
|
272
274
|
try:
|
273
|
-
response = await self._client.change_lease(
|
275
|
+
response: Any = await self._client.change_lease(
|
274
276
|
lease_id=self.id,
|
275
277
|
proposed_lease_id=proposed_lease_id,
|
276
278
|
timeout=kwargs.pop('timeout', None),
|
@@ -279,13 +281,12 @@ class BlobLeaseClient(LeaseClientBase):
|
|
279
281
|
**kwargs)
|
280
282
|
except HttpResponseError as error:
|
281
283
|
process_storage_error(error)
|
282
|
-
self.etag = response.get('etag')
|
283
|
-
self.id = response.get('lease_id')
|
284
|
-
self.last_modified = response.get('last_modified')
|
284
|
+
self.etag = response.get('etag')
|
285
|
+
self.id = response.get('lease_id')
|
286
|
+
self.last_modified = response.get('last_modified')
|
285
287
|
|
286
288
|
@distributed_trace_async
|
287
|
-
async def break_lease(self, lease_break_period=None, **kwargs):
|
288
|
-
# type: (Optional[int], Any) -> int
|
289
|
+
async def break_lease(self, lease_break_period: Optional[int] = None, **kwargs: Any) -> int:
|
289
290
|
"""Break the lease, if the container or blob has an active lease.
|
290
291
|
|
291
292
|
Once a lease is broken, it cannot be renewed. Any authorized request can break the lease;
|
@@ -334,7 +335,7 @@ class BlobLeaseClient(LeaseClientBase):
|
|
334
335
|
"""
|
335
336
|
mod_conditions = get_modify_conditions(kwargs)
|
336
337
|
try:
|
337
|
-
response = await self._client.break_lease(
|
338
|
+
response: Any = await self._client.break_lease(
|
338
339
|
timeout=kwargs.pop('timeout', None),
|
339
340
|
break_period=lease_break_period,
|
340
341
|
modified_access_conditions=mod_conditions,
|