azure-storage-blob 12.19.0b1__py3-none-any.whl → 12.20.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- azure/storage/blob/__init__.py +17 -5
- azure/storage/blob/_blob_client.py +26 -6
- azure/storage/blob/_blob_service_client.py +11 -4
- azure/storage/blob/_container_client.py +45 -17
- azure/storage/blob/_download.py +5 -4
- azure/storage/blob/_encryption.py +254 -165
- azure/storage/blob/_generated/_azure_blob_storage.py +21 -3
- azure/storage/blob/_generated/_configuration.py +4 -11
- azure/storage/blob/_generated/_serialization.py +41 -49
- azure/storage/blob/_generated/aio/_azure_blob_storage.py +23 -3
- azure/storage/blob/_generated/aio/_configuration.py +4 -11
- azure/storage/blob/_generated/aio/operations/_append_blob_operations.py +24 -58
- azure/storage/blob/_generated/aio/operations/_blob_operations.py +123 -306
- azure/storage/blob/_generated/aio/operations/_block_blob_operations.py +37 -86
- azure/storage/blob/_generated/aio/operations/_container_operations.py +98 -289
- azure/storage/blob/_generated/aio/operations/_page_blob_operations.py +51 -150
- azure/storage/blob/_generated/aio/operations/_service_operations.py +49 -125
- azure/storage/blob/_generated/models/_models_py3.py +31 -31
- azure/storage/blob/_generated/operations/_append_blob_operations.py +25 -59
- azure/storage/blob/_generated/operations/_blob_operations.py +123 -306
- azure/storage/blob/_generated/operations/_block_blob_operations.py +39 -88
- azure/storage/blob/_generated/operations/_container_operations.py +100 -291
- azure/storage/blob/_generated/operations/_page_blob_operations.py +52 -151
- azure/storage/blob/_generated/operations/_service_operations.py +50 -126
- azure/storage/blob/_lease.py +1 -0
- azure/storage/blob/_models.py +3 -4
- azure/storage/blob/_serialize.py +1 -0
- azure/storage/blob/_shared/authentication.py +1 -1
- azure/storage/blob/_shared/avro/avro_io.py +0 -6
- azure/storage/blob/_shared/avro/avro_io_async.py +0 -6
- azure/storage/blob/_shared/avro/datafile.py +0 -4
- azure/storage/blob/_shared/avro/datafile_async.py +0 -4
- azure/storage/blob/_shared/avro/schema.py +4 -4
- azure/storage/blob/_shared/base_client.py +72 -87
- azure/storage/blob/_shared/base_client_async.py +115 -27
- azure/storage/blob/_shared/models.py +120 -27
- azure/storage/blob/_shared/parser.py +7 -6
- azure/storage/blob/_shared/policies.py +96 -66
- azure/storage/blob/_shared/policies_async.py +48 -21
- azure/storage/blob/_shared/response_handlers.py +14 -16
- azure/storage/blob/_shared/shared_access_signature.py +3 -3
- azure/storage/blob/_shared_access_signature.py +48 -33
- azure/storage/blob/_upload_helpers.py +4 -7
- azure/storage/blob/_version.py +1 -1
- azure/storage/blob/aio/__init__.py +13 -4
- azure/storage/blob/aio/_blob_client_async.py +17 -6
- azure/storage/blob/aio/_blob_service_client_async.py +6 -3
- azure/storage/blob/aio/_container_client_async.py +34 -13
- azure/storage/blob/aio/_download_async.py +11 -10
- azure/storage/blob/aio/_encryption_async.py +72 -0
- azure/storage/blob/aio/_lease_async.py +1 -1
- azure/storage/blob/aio/_upload_helpers.py +8 -10
- {azure_storage_blob-12.19.0b1.dist-info → azure_storage_blob-12.20.0.dist-info}/METADATA +11 -11
- azure_storage_blob-12.20.0.dist-info/RECORD +81 -0
- {azure_storage_blob-12.19.0b1.dist-info → azure_storage_blob-12.20.0.dist-info}/WHEEL +1 -1
- azure_storage_blob-12.19.0b1.dist-info/RECORD +0 -80
- {azure_storage_blob-12.19.0b1.dist-info → azure_storage_blob-12.20.0.dist-info}/LICENSE +0 -0
- {azure_storage_blob-12.19.0b1.dist-info → azure_storage_blob-12.20.0.dist-info}/top_level.txt +0 -0
@@ -5,26 +5,22 @@
|
|
5
5
|
# --------------------------------------------------------------------------
|
6
6
|
import logging
|
7
7
|
import uuid
|
8
|
-
from typing import (
|
8
|
+
from typing import (
|
9
9
|
Any,
|
10
|
+
cast,
|
10
11
|
Dict,
|
12
|
+
Iterator,
|
11
13
|
Optional,
|
12
14
|
Tuple,
|
13
15
|
TYPE_CHECKING,
|
14
16
|
Union,
|
15
17
|
)
|
18
|
+
from urllib.parse import parse_qs, quote
|
16
19
|
|
17
|
-
|
18
|
-
from urllib.parse import parse_qs, quote
|
19
|
-
except ImportError:
|
20
|
-
from urlparse import parse_qs # type: ignore
|
21
|
-
from urllib2 import quote # type: ignore
|
22
|
-
|
23
|
-
from azure.core.configuration import Configuration
|
24
|
-
from azure.core.credentials import AzureSasCredential, AzureNamedKeyCredential
|
20
|
+
from azure.core.credentials import AzureSasCredential, AzureNamedKeyCredential, TokenCredential
|
25
21
|
from azure.core.exceptions import HttpResponseError
|
26
22
|
from azure.core.pipeline import Pipeline
|
27
|
-
from azure.core.pipeline.transport import
|
23
|
+
from azure.core.pipeline.transport import HttpTransport, RequestsTransport # pylint: disable=non-abstract-transport-import, no-name-in-module
|
28
24
|
from azure.core.pipeline.policies import (
|
29
25
|
AzureSasCredentialPolicy,
|
30
26
|
ContentDecodePolicy,
|
@@ -35,11 +31,9 @@ from azure.core.pipeline.policies import (
|
|
35
31
|
UserAgentPolicy,
|
36
32
|
)
|
37
33
|
|
38
|
-
from .constants import CONNECTION_TIMEOUT, DEFAULT_OAUTH_SCOPE, READ_TIMEOUT, SERVICE_HOST_BASE, STORAGE_OAUTH_SCOPE
|
39
|
-
from .models import LocationMode
|
40
34
|
from .authentication import SharedKeyCredentialPolicy
|
41
|
-
from .
|
42
|
-
from .
|
35
|
+
from .constants import CONNECTION_TIMEOUT, DEFAULT_OAUTH_SCOPE, READ_TIMEOUT, SERVICE_HOST_BASE, STORAGE_OAUTH_SCOPE
|
36
|
+
from .models import LocationMode, StorageConfiguration
|
43
37
|
from .policies import (
|
44
38
|
ExponentialRetry,
|
45
39
|
QueueMessagePolicy,
|
@@ -51,11 +45,15 @@ from .policies import (
|
|
51
45
|
StorageRequestHook,
|
52
46
|
StorageResponseHook,
|
53
47
|
)
|
48
|
+
from .request_handlers import serialize_batch_body, _get_batch_request_delimiter
|
49
|
+
from .response_handlers import PartialBatchErrorException, process_storage_error
|
50
|
+
from .shared_access_signature import QueryStringConstants
|
54
51
|
from .._version import VERSION
|
55
|
-
from
|
52
|
+
from .._shared_access_signature import _is_credential_sastoken
|
56
53
|
|
57
54
|
if TYPE_CHECKING:
|
58
|
-
from azure.core.
|
55
|
+
from azure.core.credentials_async import AsyncTokenCredential
|
56
|
+
from azure.core.pipeline.transport import HttpRequest, HttpResponse # pylint: disable=C4756
|
59
57
|
|
60
58
|
_LOGGER = logging.getLogger(__name__)
|
61
59
|
_SERVICE_PARAMS = {
|
@@ -67,14 +65,14 @@ _SERVICE_PARAMS = {
|
|
67
65
|
|
68
66
|
|
69
67
|
class StorageAccountHostsMixin(object): # pylint: disable=too-many-instance-attributes
|
68
|
+
_client: Any
|
70
69
|
def __init__(
|
71
70
|
self,
|
72
|
-
parsed_url
|
73
|
-
service
|
74
|
-
credential
|
75
|
-
**kwargs
|
76
|
-
):
|
77
|
-
# type: (...) -> None
|
71
|
+
parsed_url: Any,
|
72
|
+
service: str,
|
73
|
+
credential: Optional[Union[str, Dict[str, str], AzureNamedKeyCredential, AzureSasCredential, "AsyncTokenCredential", TokenCredential]] = None, # pylint: disable=line-too-long
|
74
|
+
**kwargs: Any
|
75
|
+
) -> None:
|
78
76
|
self._location_mode = kwargs.get("_location_mode", LocationMode.PRIMARY)
|
79
77
|
self._hosts = kwargs.get("_hosts")
|
80
78
|
self.scheme = parsed_url.scheme
|
@@ -137,7 +135,7 @@ class StorageAccountHostsMixin(object): # pylint: disable=too-many-instance-att
|
|
137
135
|
def primary_endpoint(self):
|
138
136
|
"""The full primary endpoint URL.
|
139
137
|
|
140
|
-
:
|
138
|
+
:rtype: str
|
141
139
|
"""
|
142
140
|
return self._format_url(self._hosts[LocationMode.PRIMARY])
|
143
141
|
|
@@ -145,7 +143,7 @@ class StorageAccountHostsMixin(object): # pylint: disable=too-many-instance-att
|
|
145
143
|
def primary_hostname(self):
|
146
144
|
"""The hostname of the primary endpoint.
|
147
145
|
|
148
|
-
:
|
146
|
+
:rtype: str
|
149
147
|
"""
|
150
148
|
return self._hosts[LocationMode.PRIMARY]
|
151
149
|
|
@@ -156,7 +154,7 @@ class StorageAccountHostsMixin(object): # pylint: disable=too-many-instance-att
|
|
156
154
|
If not available a ValueError will be raised. To explicitly specify a secondary hostname, use the optional
|
157
155
|
`secondary_hostname` keyword argument on instantiation.
|
158
156
|
|
159
|
-
:
|
157
|
+
:rtype: str
|
160
158
|
:raise ValueError:
|
161
159
|
"""
|
162
160
|
if not self._hosts[LocationMode.SECONDARY]:
|
@@ -170,7 +168,7 @@ class StorageAccountHostsMixin(object): # pylint: disable=too-many-instance-att
|
|
170
168
|
If not available this will be None. To explicitly specify a secondary hostname, use the optional
|
171
169
|
`secondary_hostname` keyword argument on instantiation.
|
172
170
|
|
173
|
-
:
|
171
|
+
:rtype: Optional[str]
|
174
172
|
"""
|
175
173
|
return self._hosts[LocationMode.SECONDARY]
|
176
174
|
|
@@ -180,7 +178,7 @@ class StorageAccountHostsMixin(object): # pylint: disable=too-many-instance-att
|
|
180
178
|
|
181
179
|
By default this will be "primary". Options include "primary" and "secondary".
|
182
180
|
|
183
|
-
:
|
181
|
+
:rtype: str
|
184
182
|
"""
|
185
183
|
|
186
184
|
return self._location_mode
|
@@ -197,29 +195,37 @@ class StorageAccountHostsMixin(object): # pylint: disable=too-many-instance-att
|
|
197
195
|
def api_version(self):
|
198
196
|
"""The version of the Storage API used for requests.
|
199
197
|
|
200
|
-
:
|
198
|
+
:rtype: str
|
201
199
|
"""
|
202
200
|
return self._client._config.version # pylint: disable=protected-access
|
203
201
|
|
204
|
-
def _format_query_string(
|
202
|
+
def _format_query_string(
|
203
|
+
self, sas_token: Optional[str],
|
204
|
+
credential: Optional[Union[str, Dict[str, str], "AzureNamedKeyCredential", "AzureSasCredential", TokenCredential]], # pylint: disable=line-too-long
|
205
|
+
snapshot: Optional[str] = None,
|
206
|
+
share_snapshot: Optional[str] = None
|
207
|
+
) -> Tuple[str, Optional[Union[str, Dict[str, str], "AzureNamedKeyCredential", "AzureSasCredential", TokenCredential]]]: # pylint: disable=line-too-long
|
205
208
|
query_str = "?"
|
206
209
|
if snapshot:
|
207
|
-
query_str += f"snapshot={
|
210
|
+
query_str += f"snapshot={snapshot}&"
|
208
211
|
if share_snapshot:
|
209
|
-
query_str += f"sharesnapshot={
|
212
|
+
query_str += f"sharesnapshot={share_snapshot}&"
|
210
213
|
if sas_token and isinstance(credential, AzureSasCredential):
|
211
214
|
raise ValueError(
|
212
215
|
"You cannot use AzureSasCredential when the resource URI also contains a Shared Access Signature.")
|
213
|
-
if
|
216
|
+
if _is_credential_sastoken(credential):
|
217
|
+
credential = cast(str, credential)
|
214
218
|
query_str += credential.lstrip("?")
|
215
219
|
credential = None
|
216
220
|
elif sas_token:
|
217
221
|
query_str += sas_token
|
218
222
|
return query_str.rstrip("?&"), credential
|
219
223
|
|
220
|
-
def _create_pipeline(
|
221
|
-
|
222
|
-
|
224
|
+
def _create_pipeline(
|
225
|
+
self, credential: Optional[Union[str, Dict[str, str], AzureNamedKeyCredential, AzureSasCredential, TokenCredential]] = None, # pylint: disable=line-too-long
|
226
|
+
**kwargs: Any
|
227
|
+
) -> Tuple[StorageConfiguration, Pipeline]:
|
228
|
+
self._credential_policy: Any = None
|
223
229
|
if hasattr(credential, "get_token"):
|
224
230
|
if kwargs.get('audience'):
|
225
231
|
audience = str(kwargs.pop('audience')).rstrip('/') + DEFAULT_OAUTH_SCOPE
|
@@ -236,11 +242,11 @@ class StorageAccountHostsMixin(object): # pylint: disable=too-many-instance-att
|
|
236
242
|
config = kwargs.get("_configuration") or create_configuration(**kwargs)
|
237
243
|
if kwargs.get("_pipeline"):
|
238
244
|
return config, kwargs["_pipeline"]
|
239
|
-
|
245
|
+
transport = kwargs.get("transport")
|
240
246
|
kwargs.setdefault("connection_timeout", CONNECTION_TIMEOUT)
|
241
247
|
kwargs.setdefault("read_timeout", READ_TIMEOUT)
|
242
|
-
if not
|
243
|
-
|
248
|
+
if not transport:
|
249
|
+
transport = RequestsTransport(**kwargs)
|
244
250
|
policies = [
|
245
251
|
QueueMessagePolicy(),
|
246
252
|
config.proxy_policy,
|
@@ -259,15 +265,21 @@ class StorageAccountHostsMixin(object): # pylint: disable=too-many-instance-att
|
|
259
265
|
HttpLoggingPolicy(**kwargs)
|
260
266
|
]
|
261
267
|
if kwargs.get("_additional_pipeline_policies"):
|
262
|
-
policies = policies + kwargs.get("_additional_pipeline_policies")
|
263
|
-
|
268
|
+
policies = policies + kwargs.get("_additional_pipeline_policies") # type: ignore
|
269
|
+
config.transport = transport # type: ignore
|
270
|
+
return config, Pipeline(transport, policies=policies)
|
264
271
|
|
265
|
-
# Given a series of request, do a Storage batch call.
|
266
272
|
def _batch_send(
|
267
273
|
self,
|
268
|
-
*reqs
|
269
|
-
**kwargs
|
270
|
-
):
|
274
|
+
*reqs: "HttpRequest",
|
275
|
+
**kwargs: Any
|
276
|
+
) -> Iterator["HttpResponse"]:
|
277
|
+
"""Given a series of request, do a Storage batch call.
|
278
|
+
|
279
|
+
:param HttpRequest reqs: A collection of HttpRequest objects.
|
280
|
+
:returns: An iterator of HttpResponse objects.
|
281
|
+
:rtype: Iterator[HttpResponse]
|
282
|
+
"""
|
271
283
|
# Pop it here, so requests doesn't feel bad about additional kwarg
|
272
284
|
raise_on_any_failure = kwargs.pop("raise_on_any_failure", True)
|
273
285
|
batch_id = str(uuid.uuid1())
|
@@ -348,7 +360,10 @@ class TransportWrapper(HttpTransport):
|
|
348
360
|
pass
|
349
361
|
|
350
362
|
|
351
|
-
def _format_shared_key_credential(
|
363
|
+
def _format_shared_key_credential(
|
364
|
+
account_name: str,
|
365
|
+
credential: Optional[Union[str, Dict[str, str], AzureNamedKeyCredential, AzureSasCredential, "AsyncTokenCredential", TokenCredential]] = None, # pylint: disable=line-too-long
|
366
|
+
) -> Any:
|
352
367
|
if isinstance(credential, str):
|
353
368
|
if not account_name:
|
354
369
|
raise ValueError("Unable to determine account name for shared key credential.")
|
@@ -364,12 +379,16 @@ def _format_shared_key_credential(account_name, credential):
|
|
364
379
|
return credential
|
365
380
|
|
366
381
|
|
367
|
-
def parse_connection_str(
|
382
|
+
def parse_connection_str(
|
383
|
+
conn_str: str,
|
384
|
+
credential: Optional[Union[str, Dict[str, str], AzureNamedKeyCredential, AzureSasCredential, TokenCredential]], # pylint: disable=line-too-long
|
385
|
+
service: str
|
386
|
+
) -> Tuple[str, Optional[str], Optional[Union[str, Dict[str, str], AzureNamedKeyCredential, AzureSasCredential, TokenCredential]]]: # pylint: disable=line-too-long
|
368
387
|
conn_str = conn_str.rstrip(";")
|
369
|
-
|
370
|
-
if any(len(tup) != 2 for tup in
|
388
|
+
conn_settings_list = [s.split("=", 1) for s in conn_str.split(";")]
|
389
|
+
if any(len(tup) != 2 for tup in conn_settings_list):
|
371
390
|
raise ValueError("Connection string is either blank or malformed.")
|
372
|
-
conn_settings = dict((key.upper(), val) for key, val in
|
391
|
+
conn_settings = dict((key.upper(), val) for key, val in conn_settings_list)
|
373
392
|
endpoints = _SERVICE_PARAMS[service]
|
374
393
|
primary = None
|
375
394
|
secondary = None
|
@@ -412,43 +431,20 @@ def parse_connection_str(conn_str, credential, service):
|
|
412
431
|
return primary, secondary, credential
|
413
432
|
|
414
433
|
|
415
|
-
def create_configuration(**kwargs):
|
416
|
-
|
417
|
-
# Backwards compatibility if someone is not passing sdk_moniker
|
434
|
+
def create_configuration(**kwargs: Any) -> StorageConfiguration:
|
435
|
+
# Backwards compatibility if someone is not passing sdk_moniker
|
418
436
|
if not kwargs.get("sdk_moniker"):
|
419
437
|
kwargs["sdk_moniker"] = f"storage-{kwargs.pop('storage_sdk')}/{VERSION}"
|
420
|
-
config =
|
438
|
+
config = StorageConfiguration(**kwargs)
|
421
439
|
config.headers_policy = StorageHeadersPolicy(**kwargs)
|
422
440
|
config.user_agent_policy = UserAgentPolicy(**kwargs)
|
423
441
|
config.retry_policy = kwargs.get("retry_policy") or ExponentialRetry(**kwargs)
|
424
442
|
config.logging_policy = StorageLoggingPolicy(**kwargs)
|
425
443
|
config.proxy_policy = ProxyPolicy(**kwargs)
|
426
|
-
|
427
|
-
# Storage settings
|
428
|
-
config.max_single_put_size = kwargs.get("max_single_put_size", 64 * 1024 * 1024)
|
429
|
-
config.copy_polling_interval = 15
|
430
|
-
|
431
|
-
# Block blob uploads
|
432
|
-
config.max_block_size = kwargs.get("max_block_size", 4 * 1024 * 1024)
|
433
|
-
config.min_large_block_upload_threshold = kwargs.get("min_large_block_upload_threshold", 4 * 1024 * 1024 + 1)
|
434
|
-
config.use_byte_buffer = kwargs.get("use_byte_buffer", False)
|
435
|
-
|
436
|
-
# Page blob uploads
|
437
|
-
config.max_page_size = kwargs.get("max_page_size", 4 * 1024 * 1024)
|
438
|
-
|
439
|
-
# Datalake file uploads
|
440
|
-
config.min_large_chunk_upload_threshold = kwargs.get("min_large_chunk_upload_threshold", 100 * 1024 * 1024 + 1)
|
441
|
-
|
442
|
-
# Blob downloads
|
443
|
-
config.max_single_get_size = kwargs.get("max_single_get_size", 32 * 1024 * 1024)
|
444
|
-
config.max_chunk_get_size = kwargs.get("max_chunk_get_size", 4 * 1024 * 1024)
|
445
|
-
|
446
|
-
# File uploads
|
447
|
-
config.max_range_size = kwargs.get("max_range_size", 4 * 1024 * 1024)
|
448
444
|
return config
|
449
445
|
|
450
446
|
|
451
|
-
def parse_query(query_str):
|
447
|
+
def parse_query(query_str: str) -> Tuple[Optional[str], Optional[str]]:
|
452
448
|
sas_values = QueryStringConstants.to_list()
|
453
449
|
parsed_query = {k: v[0] for k, v in parse_qs(query_str).items()}
|
454
450
|
sas_params = [f"{k}={quote(v, safe='')}" for k, v in parsed_query.items() if k in sas_values]
|
@@ -458,14 +454,3 @@ def parse_query(query_str):
|
|
458
454
|
|
459
455
|
snapshot = parsed_query.get("snapshot") or parsed_query.get("sharesnapshot")
|
460
456
|
return snapshot, sas_token
|
461
|
-
|
462
|
-
|
463
|
-
def is_credential_sastoken(credential):
|
464
|
-
if not credential or not isinstance(credential, str):
|
465
|
-
return False
|
466
|
-
|
467
|
-
sas_values = QueryStringConstants.to_list()
|
468
|
-
parsed_query = parse_qs(credential.lstrip("?"))
|
469
|
-
if parsed_query and all(k in sas_values for k in parsed_query.keys()):
|
470
|
-
return True
|
471
|
-
return False
|
@@ -3,17 +3,16 @@
|
|
3
3
|
# Licensed under the MIT License. See License.txt in the project root for
|
4
4
|
# license information.
|
5
5
|
# --------------------------------------------------------------------------
|
6
|
+
# mypy: disable-error-code="attr-defined"
|
6
7
|
|
7
|
-
from typing import ( # pylint: disable=unused-import
|
8
|
-
Union, Optional, Any, Iterable, Dict, List, Type, Tuple,
|
9
|
-
TYPE_CHECKING
|
10
|
-
)
|
11
8
|
import logging
|
9
|
+
from typing import Any, Dict, Optional, Tuple, TYPE_CHECKING, Union
|
12
10
|
|
13
|
-
from azure.core.credentials import AzureSasCredential
|
14
|
-
from azure.core.pipeline import AsyncPipeline
|
15
11
|
from azure.core.async_paging import AsyncList
|
12
|
+
from azure.core.credentials import AzureNamedKeyCredential, AzureSasCredential
|
13
|
+
from azure.core.credentials_async import AsyncTokenCredential
|
16
14
|
from azure.core.exceptions import HttpResponseError
|
15
|
+
from azure.core.pipeline import AsyncPipeline
|
17
16
|
from azure.core.pipeline.policies import (
|
18
17
|
AsyncRedirectPolicy,
|
19
18
|
AzureSasCredentialPolicy,
|
@@ -23,9 +22,10 @@ from azure.core.pipeline.policies import (
|
|
23
22
|
)
|
24
23
|
from azure.core.pipeline.transport import AsyncHttpTransport
|
25
24
|
|
26
|
-
from .constants import CONNECTION_TIMEOUT, DEFAULT_OAUTH_SCOPE, READ_TIMEOUT, STORAGE_OAUTH_SCOPE
|
27
25
|
from .authentication import SharedKeyCredentialPolicy
|
28
26
|
from .base_client import create_configuration
|
27
|
+
from .constants import CONNECTION_TIMEOUT, DEFAULT_OAUTH_SCOPE, READ_TIMEOUT, SERVICE_HOST_BASE, STORAGE_OAUTH_SCOPE
|
28
|
+
from .models import StorageConfiguration
|
29
29
|
from .policies import (
|
30
30
|
QueueMessagePolicy,
|
31
31
|
StorageContentValidation,
|
@@ -34,15 +34,20 @@ from .policies import (
|
|
34
34
|
StorageRequestHook,
|
35
35
|
)
|
36
36
|
from .policies_async import AsyncStorageBearerTokenCredentialPolicy, AsyncStorageResponseHook
|
37
|
-
|
38
|
-
from
|
37
|
+
from .response_handlers import PartialBatchErrorException, process_storage_error
|
38
|
+
from .._shared_access_signature import _is_credential_sastoken
|
39
39
|
|
40
40
|
if TYPE_CHECKING:
|
41
|
-
from azure.core.pipeline import
|
42
|
-
from azure.core.pipeline.transport import HttpRequest
|
43
|
-
from azure.core.configuration import Configuration
|
41
|
+
from azure.core.pipeline.transport import HttpRequest, HttpResponse # pylint: disable=C4756
|
44
42
|
_LOGGER = logging.getLogger(__name__)
|
45
43
|
|
44
|
+
_SERVICE_PARAMS = {
|
45
|
+
"blob": {"primary": "BLOBENDPOINT", "secondary": "BLOBSECONDARYENDPOINT"},
|
46
|
+
"queue": {"primary": "QUEUEENDPOINT", "secondary": "QUEUESECONDARYENDPOINT"},
|
47
|
+
"file": {"primary": "FILEENDPOINT", "secondary": "FILESECONDARYENDPOINT"},
|
48
|
+
"dfs": {"primary": "BLOBENDPOINT", "secondary": "BLOBENDPOINT"},
|
49
|
+
}
|
50
|
+
|
46
51
|
|
47
52
|
class AsyncStorageAccountHostsMixin(object):
|
48
53
|
|
@@ -65,10 +70,36 @@ class AsyncStorageAccountHostsMixin(object):
|
|
65
70
|
"""
|
66
71
|
await self._client.close()
|
67
72
|
|
68
|
-
def
|
69
|
-
|
70
|
-
|
71
|
-
|
73
|
+
def _format_query_string(
|
74
|
+
self, sas_token: Optional[str],
|
75
|
+
credential: Optional[Union[str, Dict[str, str], "AzureNamedKeyCredential", "AzureSasCredential", AsyncTokenCredential]], # pylint: disable=line-too-long
|
76
|
+
snapshot: Optional[str] = None,
|
77
|
+
share_snapshot: Optional[str] = None
|
78
|
+
) -> Tuple[str, Optional[Union[str, Dict[str, str], "AzureNamedKeyCredential", "AzureSasCredential", AsyncTokenCredential]]]: # pylint: disable=line-too-long
|
79
|
+
query_str = "?"
|
80
|
+
if snapshot:
|
81
|
+
query_str += f"snapshot={snapshot}&"
|
82
|
+
if share_snapshot:
|
83
|
+
query_str += f"sharesnapshot={share_snapshot}&"
|
84
|
+
if sas_token and isinstance(credential, AzureSasCredential):
|
85
|
+
raise ValueError(
|
86
|
+
"You cannot use AzureSasCredential when the resource URI also contains a Shared Access Signature.")
|
87
|
+
if _is_credential_sastoken(credential):
|
88
|
+
query_str += credential.lstrip("?") # type: ignore [union-attr]
|
89
|
+
credential = None
|
90
|
+
elif sas_token:
|
91
|
+
query_str += sas_token
|
92
|
+
return query_str.rstrip("?&"), credential
|
93
|
+
|
94
|
+
def _create_pipeline(
|
95
|
+
self, credential: Optional[Union[str, Dict[str, str], AzureNamedKeyCredential, AzureSasCredential, AsyncTokenCredential]] = None, # pylint: disable=line-too-long
|
96
|
+
**kwargs: Any
|
97
|
+
) -> Tuple[StorageConfiguration, AsyncPipeline]:
|
98
|
+
self._credential_policy: Optional[
|
99
|
+
Union[AsyncStorageBearerTokenCredentialPolicy,
|
100
|
+
SharedKeyCredentialPolicy,
|
101
|
+
AzureSasCredentialPolicy]] = None
|
102
|
+
if hasattr(credential, 'get_token'):
|
72
103
|
if kwargs.get('audience'):
|
73
104
|
audience = str(kwargs.pop('audience')).rstrip('/') + DEFAULT_OAUTH_SCOPE
|
74
105
|
else:
|
@@ -83,15 +114,16 @@ class AsyncStorageAccountHostsMixin(object):
|
|
83
114
|
config = kwargs.get('_configuration') or create_configuration(**kwargs)
|
84
115
|
if kwargs.get('_pipeline'):
|
85
116
|
return config, kwargs['_pipeline']
|
86
|
-
|
117
|
+
transport = kwargs.get('transport')
|
87
118
|
kwargs.setdefault("connection_timeout", CONNECTION_TIMEOUT)
|
88
119
|
kwargs.setdefault("read_timeout", READ_TIMEOUT)
|
89
|
-
if not
|
120
|
+
if not transport:
|
90
121
|
try:
|
91
122
|
from azure.core.pipeline.transport import AioHttpTransport # pylint: disable=non-abstract-transport-import
|
92
123
|
except ImportError as exc:
|
93
124
|
raise ImportError("Unable to create async transport. Please check aiohttp is installed.") from exc
|
94
|
-
|
125
|
+
transport = AioHttpTransport(**kwargs)
|
126
|
+
hosts = self._hosts
|
95
127
|
policies = [
|
96
128
|
QueueMessagePolicy(),
|
97
129
|
config.headers_policy,
|
@@ -102,7 +134,7 @@ class AsyncStorageAccountHostsMixin(object):
|
|
102
134
|
self._credential_policy,
|
103
135
|
ContentDecodePolicy(response_encoding="utf-8"),
|
104
136
|
AsyncRedirectPolicy(**kwargs),
|
105
|
-
StorageHosts(hosts=
|
137
|
+
StorageHosts(hosts=hosts, **kwargs),
|
106
138
|
config.retry_policy,
|
107
139
|
config.logging_policy,
|
108
140
|
AsyncStorageResponseHook(**kwargs),
|
@@ -110,15 +142,21 @@ class AsyncStorageAccountHostsMixin(object):
|
|
110
142
|
HttpLoggingPolicy(**kwargs),
|
111
143
|
]
|
112
144
|
if kwargs.get("_additional_pipeline_policies"):
|
113
|
-
policies = policies + kwargs.get("_additional_pipeline_policies")
|
114
|
-
|
145
|
+
policies = policies + kwargs.get("_additional_pipeline_policies") #type: ignore
|
146
|
+
config.transport = transport #type: ignore
|
147
|
+
return config, AsyncPipeline(transport, policies=policies) #type: ignore
|
115
148
|
|
116
|
-
# Given a series of request, do a Storage batch call.
|
117
149
|
async def _batch_send(
|
118
150
|
self,
|
119
|
-
*reqs
|
120
|
-
**kwargs
|
121
|
-
):
|
151
|
+
*reqs: "HttpRequest",
|
152
|
+
**kwargs: Any
|
153
|
+
) -> AsyncList["HttpResponse"]:
|
154
|
+
"""Given a series of request, do a Storage batch call.
|
155
|
+
|
156
|
+
:param HttpRequest reqs: A collection of HttpRequest objects.
|
157
|
+
:returns: An AsyncList of HttpResponse objects.
|
158
|
+
:rtype: AsyncList[HttpResponse]
|
159
|
+
"""
|
122
160
|
# Pop it here, so requests doesn't feel bad about additional kwarg
|
123
161
|
raise_on_any_failure = kwargs.pop("raise_on_any_failure", True)
|
124
162
|
request = self._client._client.post( # pylint: disable=protected-access
|
@@ -134,7 +172,7 @@ class AsyncStorageAccountHostsMixin(object):
|
|
134
172
|
|
135
173
|
policies = [StorageHeadersPolicy()]
|
136
174
|
if self._credential_policy:
|
137
|
-
policies.append(self._credential_policy)
|
175
|
+
policies.append(self._credential_policy) # type: ignore
|
138
176
|
|
139
177
|
request.set_multipart_mixed(
|
140
178
|
*reqs,
|
@@ -166,6 +204,56 @@ class AsyncStorageAccountHostsMixin(object):
|
|
166
204
|
except HttpResponseError as error:
|
167
205
|
process_storage_error(error)
|
168
206
|
|
207
|
+
def parse_connection_str(
|
208
|
+
conn_str: str,
|
209
|
+
credential: Optional[Union[str, Dict[str, str], AzureNamedKeyCredential, AzureSasCredential, AsyncTokenCredential]], # pylint: disable=line-too-long
|
210
|
+
service: str
|
211
|
+
) -> Tuple[str, Optional[str], Optional[Union[str, Dict[str, str], AzureNamedKeyCredential, AzureSasCredential, AsyncTokenCredential]]]: # pylint: disable=line-too-long
|
212
|
+
conn_str = conn_str.rstrip(";")
|
213
|
+
conn_settings_list = [s.split("=", 1) for s in conn_str.split(";")]
|
214
|
+
if any(len(tup) != 2 for tup in conn_settings_list):
|
215
|
+
raise ValueError("Connection string is either blank or malformed.")
|
216
|
+
conn_settings = dict((key.upper(), val) for key, val in conn_settings_list)
|
217
|
+
endpoints = _SERVICE_PARAMS[service]
|
218
|
+
primary = None
|
219
|
+
secondary = None
|
220
|
+
if not credential:
|
221
|
+
try:
|
222
|
+
credential = {"account_name": conn_settings["ACCOUNTNAME"], "account_key": conn_settings["ACCOUNTKEY"]}
|
223
|
+
except KeyError:
|
224
|
+
credential = conn_settings.get("SHAREDACCESSSIGNATURE")
|
225
|
+
if endpoints["primary"] in conn_settings:
|
226
|
+
primary = conn_settings[endpoints["primary"]]
|
227
|
+
if endpoints["secondary"] in conn_settings:
|
228
|
+
secondary = conn_settings[endpoints["secondary"]]
|
229
|
+
else:
|
230
|
+
if endpoints["secondary"] in conn_settings:
|
231
|
+
raise ValueError("Connection string specifies only secondary endpoint.")
|
232
|
+
try:
|
233
|
+
primary =(
|
234
|
+
f"{conn_settings['DEFAULTENDPOINTSPROTOCOL']}://"
|
235
|
+
f"{conn_settings['ACCOUNTNAME']}.{service}.{conn_settings['ENDPOINTSUFFIX']}"
|
236
|
+
)
|
237
|
+
secondary = (
|
238
|
+
f"{conn_settings['ACCOUNTNAME']}-secondary."
|
239
|
+
f"{service}.{conn_settings['ENDPOINTSUFFIX']}"
|
240
|
+
)
|
241
|
+
except KeyError:
|
242
|
+
pass
|
243
|
+
|
244
|
+
if not primary:
|
245
|
+
try:
|
246
|
+
primary = (
|
247
|
+
f"https://{conn_settings['ACCOUNTNAME']}."
|
248
|
+
f"{service}.{conn_settings.get('ENDPOINTSUFFIX', SERVICE_HOST_BASE)}"
|
249
|
+
)
|
250
|
+
except KeyError as exc:
|
251
|
+
raise ValueError("Connection string missing required connection details.") from exc
|
252
|
+
if service == "dfs":
|
253
|
+
primary = primary.replace(".blob.", ".dfs.")
|
254
|
+
if secondary:
|
255
|
+
secondary = secondary.replace(".blob.", ".dfs.")
|
256
|
+
return primary, secondary, credential
|
169
257
|
|
170
258
|
class AsyncTransportWrapper(AsyncHttpTransport):
|
171
259
|
"""Wrapper class that ensures that an inner client created
|