azure-storage-blob 12.25.1__py3-none-any.whl → 12.27.0b1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- azure/storage/blob/__init__.py +3 -2
- azure/storage/blob/_blob_client.py +94 -41
- azure/storage/blob/_blob_client_helpers.py +19 -4
- azure/storage/blob/_blob_service_client.py +16 -13
- azure/storage/blob/_container_client.py +25 -22
- azure/storage/blob/_deserialize.py +1 -1
- azure/storage/blob/_download.py +7 -7
- azure/storage/blob/_encryption.py +177 -184
- azure/storage/blob/_generated/_azure_blob_storage.py +3 -2
- azure/storage/blob/_generated/_configuration.py +2 -2
- azure/storage/blob/_generated/_utils/__init__.py +6 -0
- azure/storage/blob/_generated/{_serialization.py → _utils/serialization.py} +7 -25
- azure/storage/blob/_generated/aio/_azure_blob_storage.py +3 -2
- azure/storage/blob/_generated/aio/_configuration.py +2 -2
- azure/storage/blob/_generated/aio/operations/_append_blob_operations.py +11 -14
- azure/storage/blob/_generated/aio/operations/_blob_operations.py +40 -64
- azure/storage/blob/_generated/aio/operations/_block_blob_operations.py +18 -20
- azure/storage/blob/_generated/aio/operations/_container_operations.py +21 -43
- azure/storage/blob/_generated/aio/operations/_page_blob_operations.py +18 -27
- azure/storage/blob/_generated/aio/operations/_service_operations.py +11 -22
- azure/storage/blob/_generated/models/__init__.py +2 -0
- azure/storage/blob/_generated/models/_azure_blob_storage_enums.py +6 -0
- azure/storage/blob/_generated/models/_models_py3.py +30 -9
- azure/storage/blob/_generated/operations/_append_blob_operations.py +19 -20
- azure/storage/blob/_generated/operations/_blob_operations.py +68 -89
- azure/storage/blob/_generated/operations/_block_blob_operations.py +31 -27
- azure/storage/blob/_generated/operations/_container_operations.py +40 -62
- azure/storage/blob/_generated/operations/_page_blob_operations.py +31 -37
- azure/storage/blob/_generated/operations/_service_operations.py +20 -32
- azure/storage/blob/_lease.py +1 -0
- azure/storage/blob/_list_blobs_helper.py +1 -1
- azure/storage/blob/_quick_query_helper.py +20 -24
- azure/storage/blob/_serialize.py +2 -0
- azure/storage/blob/_shared/__init__.py +7 -7
- azure/storage/blob/_shared/authentication.py +49 -32
- azure/storage/blob/_shared/avro/avro_io.py +44 -42
- azure/storage/blob/_shared/avro/avro_io_async.py +42 -41
- azure/storage/blob/_shared/avro/datafile.py +24 -21
- azure/storage/blob/_shared/avro/datafile_async.py +15 -15
- azure/storage/blob/_shared/avro/schema.py +196 -217
- azure/storage/blob/_shared/base_client.py +82 -59
- azure/storage/blob/_shared/base_client_async.py +58 -51
- azure/storage/blob/_shared/constants.py +1 -1
- azure/storage/blob/_shared/models.py +94 -92
- azure/storage/blob/_shared/parser.py +3 -3
- azure/storage/blob/_shared/policies.py +186 -147
- azure/storage/blob/_shared/policies_async.py +53 -65
- azure/storage/blob/_shared/request_handlers.py +50 -45
- azure/storage/blob/_shared/response_handlers.py +54 -45
- azure/storage/blob/_shared/shared_access_signature.py +67 -71
- azure/storage/blob/_shared/uploads.py +56 -49
- azure/storage/blob/_shared/uploads_async.py +70 -58
- azure/storage/blob/_shared_access_signature.py +3 -1
- azure/storage/blob/_version.py +1 -1
- azure/storage/blob/aio/__init__.py +3 -2
- azure/storage/blob/aio/_blob_client_async.py +241 -44
- azure/storage/blob/aio/_blob_service_client_async.py +13 -11
- azure/storage/blob/aio/_container_client_async.py +28 -25
- azure/storage/blob/aio/_download_async.py +7 -7
- azure/storage/blob/aio/_lease_async.py +1 -0
- azure/storage/blob/aio/_quick_query_helper_async.py +194 -0
- {azure_storage_blob-12.25.1.dist-info → azure_storage_blob-12.27.0b1.dist-info}/METADATA +4 -5
- azure_storage_blob-12.27.0b1.dist-info/RECORD +86 -0
- azure_storage_blob-12.25.1.dist-info/RECORD +0 -84
- {azure_storage_blob-12.25.1.dist-info → azure_storage_blob-12.27.0b1.dist-info}/LICENSE +0 -0
- {azure_storage_blob-12.25.1.dist-info → azure_storage_blob-12.27.0b1.dist-info}/WHEEL +0 -0
- {azure_storage_blob-12.25.1.dist-info → azure_storage_blob-12.27.0b1.dist-info}/top_level.txt +0 -0
@@ -38,51 +38,46 @@ if TYPE_CHECKING:
|
|
38
38
|
from cryptography.hazmat.primitives.padding import PaddingContext
|
39
39
|
|
40
40
|
|
41
|
-
_ENCRYPTION_PROTOCOL_V1 =
|
42
|
-
_ENCRYPTION_PROTOCOL_V2 =
|
43
|
-
_ENCRYPTION_PROTOCOL_V2_1 =
|
41
|
+
_ENCRYPTION_PROTOCOL_V1 = "1.0"
|
42
|
+
_ENCRYPTION_PROTOCOL_V2 = "2.0"
|
43
|
+
_ENCRYPTION_PROTOCOL_V2_1 = "2.1"
|
44
44
|
_VALID_ENCRYPTION_PROTOCOLS = [_ENCRYPTION_PROTOCOL_V1, _ENCRYPTION_PROTOCOL_V2, _ENCRYPTION_PROTOCOL_V2_1]
|
45
45
|
_ENCRYPTION_V2_PROTOCOLS = [_ENCRYPTION_PROTOCOL_V2, _ENCRYPTION_PROTOCOL_V2_1]
|
46
46
|
_GCM_REGION_DATA_LENGTH = 4 * 1024 * 1024
|
47
47
|
_GCM_NONCE_LENGTH = 12
|
48
48
|
_GCM_TAG_LENGTH = 16
|
49
49
|
|
50
|
-
_ERROR_OBJECT_INVALID =
|
51
|
-
'{0} does not define a complete interface. Value of {1} is either missing or invalid.'
|
50
|
+
_ERROR_OBJECT_INVALID = "{0} does not define a complete interface. Value of {1} is either missing or invalid."
|
52
51
|
|
53
52
|
_ERROR_UNSUPPORTED_METHOD_FOR_ENCRYPTION = (
|
54
|
-
|
55
|
-
|
53
|
+
"The require_encryption flag is set, but encryption is not supported for this method."
|
54
|
+
)
|
56
55
|
|
57
56
|
|
58
57
|
class KeyEncryptionKey(Protocol):
|
59
58
|
|
60
|
-
def wrap_key(self, key: bytes) -> bytes:
|
61
|
-
...
|
59
|
+
def wrap_key(self, key: bytes) -> bytes: ...
|
62
60
|
|
63
|
-
def unwrap_key(self, key: bytes, algorithm: str) -> bytes:
|
64
|
-
...
|
61
|
+
def unwrap_key(self, key: bytes, algorithm: str) -> bytes: ...
|
65
62
|
|
66
|
-
def get_kid(self) -> str:
|
67
|
-
...
|
63
|
+
def get_kid(self) -> str: ...
|
68
64
|
|
69
|
-
def get_key_wrap_algorithm(self) -> str:
|
70
|
-
...
|
65
|
+
def get_key_wrap_algorithm(self) -> str: ...
|
71
66
|
|
72
67
|
|
73
68
|
def _validate_not_none(param_name: str, param: Any):
|
74
69
|
if param is None:
|
75
|
-
raise ValueError(f
|
70
|
+
raise ValueError(f"{param_name} should not be None.")
|
76
71
|
|
77
72
|
|
78
73
|
def _validate_key_encryption_key_wrap(kek: KeyEncryptionKey):
|
79
74
|
# Note that None is not callable and so will fail the second clause of each check.
|
80
|
-
if not hasattr(kek,
|
81
|
-
raise AttributeError(_ERROR_OBJECT_INVALID.format(
|
82
|
-
if not hasattr(kek,
|
83
|
-
raise AttributeError(_ERROR_OBJECT_INVALID.format(
|
84
|
-
if not hasattr(kek,
|
85
|
-
raise AttributeError(_ERROR_OBJECT_INVALID.format(
|
75
|
+
if not hasattr(kek, "wrap_key") or not callable(kek.wrap_key):
|
76
|
+
raise AttributeError(_ERROR_OBJECT_INVALID.format("key encryption key", "wrap_key"))
|
77
|
+
if not hasattr(kek, "get_kid") or not callable(kek.get_kid):
|
78
|
+
raise AttributeError(_ERROR_OBJECT_INVALID.format("key encryption key", "get_kid"))
|
79
|
+
if not hasattr(kek, "get_key_wrap_algorithm") or not callable(kek.get_key_wrap_algorithm):
|
80
|
+
raise AttributeError(_ERROR_OBJECT_INVALID.format("key encryption key", "get_key_wrap_algorithm"))
|
86
81
|
|
87
82
|
|
88
83
|
class StorageEncryptionMixin(object):
|
@@ -91,19 +86,22 @@ class StorageEncryptionMixin(object):
|
|
91
86
|
self.encryption_version = kwargs.get("encryption_version", "1.0")
|
92
87
|
self.key_encryption_key = kwargs.get("key_encryption_key")
|
93
88
|
self.key_resolver_function = kwargs.get("key_resolver_function")
|
94
|
-
if self.key_encryption_key and self.encryption_version ==
|
95
|
-
warnings.warn(
|
96
|
-
|
97
|
-
|
98
|
-
|
89
|
+
if self.key_encryption_key and self.encryption_version == "1.0":
|
90
|
+
warnings.warn(
|
91
|
+
"This client has been configured to use encryption with version 1.0. "
|
92
|
+
+ "Version 1.0 is deprecated and no longer considered secure. It is highly "
|
93
|
+
+ "recommended that you switch to using version 2.0. The version can be "
|
94
|
+
+ "specified using the 'encryption_version' keyword."
|
95
|
+
)
|
99
96
|
|
100
97
|
|
101
98
|
class _EncryptionAlgorithm(object):
|
102
99
|
"""
|
103
100
|
Specifies which client encryption algorithm is used.
|
104
101
|
"""
|
105
|
-
|
106
|
-
|
102
|
+
|
103
|
+
AES_CBC_256 = "AES_CBC_256"
|
104
|
+
AES_GCM_256 = "AES_GCM_256"
|
107
105
|
|
108
106
|
|
109
107
|
class _WrappedContentKey:
|
@@ -120,9 +118,9 @@ class _WrappedContentKey:
|
|
120
118
|
:param str key_id:
|
121
119
|
The key-encryption-key identifier string.
|
122
120
|
"""
|
123
|
-
_validate_not_none(
|
124
|
-
_validate_not_none(
|
125
|
-
_validate_not_none(
|
121
|
+
_validate_not_none("algorithm", algorithm)
|
122
|
+
_validate_not_none("encrypted_key", encrypted_key)
|
123
|
+
_validate_not_none("key_id", key_id)
|
126
124
|
|
127
125
|
self.algorithm = algorithm
|
128
126
|
self.encrypted_key = encrypted_key
|
@@ -144,9 +142,9 @@ class _EncryptedRegionInfo:
|
|
144
142
|
:param int tag_length:
|
145
143
|
The length of the encryption tag.
|
146
144
|
"""
|
147
|
-
_validate_not_none(
|
148
|
-
_validate_not_none(
|
149
|
-
_validate_not_none(
|
145
|
+
_validate_not_none("data_length", data_length)
|
146
|
+
_validate_not_none("nonce_length", nonce_length)
|
147
|
+
_validate_not_none("tag_length", tag_length)
|
150
148
|
|
151
149
|
self.data_length = data_length
|
152
150
|
self.nonce_length = nonce_length
|
@@ -166,8 +164,8 @@ class _EncryptionAgent:
|
|
166
164
|
:param str protocol:
|
167
165
|
The protocol version used for encryption.
|
168
166
|
"""
|
169
|
-
_validate_not_none(
|
170
|
-
_validate_not_none(
|
167
|
+
_validate_not_none("encryption_algorithm", encryption_algorithm)
|
168
|
+
_validate_not_none("protocol", protocol)
|
171
169
|
|
172
170
|
self.encryption_algorithm = str(encryption_algorithm)
|
173
171
|
self.protocol = protocol
|
@@ -179,11 +177,12 @@ class _EncryptionData:
|
|
179
177
|
"""
|
180
178
|
|
181
179
|
def __init__(
|
182
|
-
self,
|
180
|
+
self,
|
181
|
+
content_encryption_IV: Optional[bytes],
|
183
182
|
encrypted_region_info: Optional[_EncryptedRegionInfo],
|
184
183
|
encryption_agent: _EncryptionAgent,
|
185
184
|
wrapped_content_key: _WrappedContentKey,
|
186
|
-
key_wrapping_metadata: Dict[str, Any]
|
185
|
+
key_wrapping_metadata: Dict[str, Any],
|
187
186
|
) -> None:
|
188
187
|
"""
|
189
188
|
:param Optional[bytes] content_encryption_IV:
|
@@ -200,14 +199,14 @@ class _EncryptionData:
|
|
200
199
|
:param Dict[str, Any] key_wrapping_metadata:
|
201
200
|
A dict containing metadata related to the key wrapping.
|
202
201
|
"""
|
203
|
-
_validate_not_none(
|
204
|
-
_validate_not_none(
|
202
|
+
_validate_not_none("encryption_agent", encryption_agent)
|
203
|
+
_validate_not_none("wrapped_content_key", wrapped_content_key)
|
205
204
|
|
206
205
|
# Validate we have the right matching optional parameter for the specified algorithm
|
207
206
|
if encryption_agent.encryption_algorithm == _EncryptionAlgorithm.AES_CBC_256:
|
208
|
-
_validate_not_none(
|
207
|
+
_validate_not_none("content_encryption_IV", content_encryption_IV)
|
209
208
|
elif encryption_agent.encryption_algorithm == _EncryptionAlgorithm.AES_GCM_256:
|
210
|
-
_validate_not_none(
|
209
|
+
_validate_not_none("encrypted_region_info", encrypted_region_info)
|
211
210
|
else:
|
212
211
|
raise ValueError("Invalid encryption algorithm.")
|
213
212
|
|
@@ -225,8 +224,10 @@ class GCMBlobEncryptionStream:
|
|
225
224
|
will use the same encryption key and will generate a guaranteed unique
|
226
225
|
nonce for each encryption region.
|
227
226
|
"""
|
227
|
+
|
228
228
|
def __init__(
|
229
|
-
self,
|
229
|
+
self,
|
230
|
+
content_encryption_key: bytes,
|
230
231
|
data_stream: IO[bytes],
|
231
232
|
) -> None:
|
232
233
|
"""
|
@@ -237,7 +238,7 @@ class GCMBlobEncryptionStream:
|
|
237
238
|
self.data_stream = data_stream
|
238
239
|
|
239
240
|
self.offset = 0
|
240
|
-
self.current = b
|
241
|
+
self.current = b""
|
241
242
|
self.nonce_counter = 0
|
242
243
|
|
243
244
|
def read(self, size: int = -1) -> bytes:
|
@@ -286,7 +287,7 @@ def encrypt_data_v2(data: bytes, nonce: int, key: bytes) -> bytes:
|
|
286
287
|
:return: The encrypted bytes in the form: nonce + ciphertext + tag.
|
287
288
|
:rtype: bytes
|
288
289
|
"""
|
289
|
-
nonce_bytes = nonce.to_bytes(_GCM_NONCE_LENGTH,
|
290
|
+
nonce_bytes = nonce.to_bytes(_GCM_NONCE_LENGTH, "big")
|
290
291
|
aesgcm = AESGCM(key)
|
291
292
|
|
292
293
|
# Returns ciphertext + tag
|
@@ -307,11 +308,8 @@ def is_encryption_v2(encryption_data: Optional[_EncryptionData]) -> bool:
|
|
307
308
|
|
308
309
|
|
309
310
|
def modify_user_agent_for_encryption(
|
310
|
-
|
311
|
-
|
312
|
-
encryption_version: str,
|
313
|
-
request_options: Dict[str, Any]
|
314
|
-
) -> None:
|
311
|
+
user_agent: str, moniker: str, encryption_version: str, request_options: Dict[str, Any]
|
312
|
+
) -> None:
|
315
313
|
"""
|
316
314
|
Modifies the request options to contain a user agent string updated with encryption information.
|
317
315
|
Adds azstorage-clientsideencryption/<version> immediately proceeding the SDK descriptor.
|
@@ -322,7 +320,7 @@ def modify_user_agent_for_encryption(
|
|
322
320
|
:param Dict[str, Any] request_options: The reuqest options to add the user agent override to.
|
323
321
|
"""
|
324
322
|
# If the user has specified user_agent_overwrite=True, don't make any modifications
|
325
|
-
if request_options.get(
|
323
|
+
if request_options.get("user_agent_overwrite"):
|
326
324
|
return
|
327
325
|
|
328
326
|
# If the feature flag is already present, don't add it again
|
@@ -333,11 +331,11 @@ def modify_user_agent_for_encryption(
|
|
333
331
|
index = user_agent.find(f"azsdk-python-{moniker}")
|
334
332
|
user_agent = f"{user_agent[:index]}{feature_flag} {user_agent[index:]}"
|
335
333
|
# Since we are using user_agent_overwrite=True, we must prepend the user's user_agent if there is one
|
336
|
-
if request_options.get(
|
334
|
+
if request_options.get("user_agent"):
|
337
335
|
user_agent = f"{request_options.get('user_agent')} {user_agent}"
|
338
336
|
|
339
|
-
request_options[
|
340
|
-
request_options[
|
337
|
+
request_options["user_agent"] = user_agent
|
338
|
+
request_options["user_agent_overwrite"] = True
|
341
339
|
|
342
340
|
|
343
341
|
def get_adjusted_upload_size(length: int, encryption_version: str) -> int:
|
@@ -362,10 +360,8 @@ def get_adjusted_upload_size(length: int, encryption_version: str) -> int:
|
|
362
360
|
|
363
361
|
|
364
362
|
def get_adjusted_download_range_and_offset(
|
365
|
-
|
366
|
-
|
367
|
-
length: Optional[int],
|
368
|
-
encryption_data: Optional[_EncryptionData]) -> Tuple[Tuple[int, int], Tuple[int, int]]:
|
363
|
+
start: int, end: int, length: Optional[int], encryption_data: Optional[_EncryptionData]
|
364
|
+
) -> Tuple[Tuple[int, int], Tuple[int, int]]:
|
369
365
|
"""
|
370
366
|
Gets the new download range and offsets into the decrypted data for
|
371
367
|
the given user-specified range. The new download range will include all
|
@@ -453,7 +449,7 @@ def parse_encryption_data(metadata: Dict[str, Any]) -> Optional[_EncryptionData]
|
|
453
449
|
try:
|
454
450
|
# Use case insensitive dict as key needs to be case-insensitive
|
455
451
|
case_insensitive_metadata = CaseInsensitiveDict(metadata)
|
456
|
-
return _dict_to_encryption_data(loads(case_insensitive_metadata[
|
452
|
+
return _dict_to_encryption_data(loads(case_insensitive_metadata["encryptiondata"]))
|
457
453
|
except: # pylint: disable=bare-except
|
458
454
|
return None
|
459
455
|
|
@@ -468,9 +464,11 @@ def adjust_blob_size_for_encryption(size: int, encryption_data: Optional[_Encryp
|
|
468
464
|
:return: The new blob size.
|
469
465
|
:rtype: int
|
470
466
|
"""
|
471
|
-
if (
|
472
|
-
encryption_data
|
473
|
-
|
467
|
+
if (
|
468
|
+
encryption_data is not None
|
469
|
+
and encryption_data.encrypted_region_info is not None
|
470
|
+
and is_encryption_v2(encryption_data)
|
471
|
+
):
|
474
472
|
|
475
473
|
nonce_length = encryption_data.encrypted_region_info.nonce_length
|
476
474
|
data_length = encryption_data.encrypted_region_info.data_length
|
@@ -485,11 +483,8 @@ def adjust_blob_size_for_encryption(size: int, encryption_data: Optional[_Encryp
|
|
485
483
|
|
486
484
|
|
487
485
|
def _generate_encryption_data_dict(
|
488
|
-
|
489
|
-
|
490
|
-
iv: Optional[bytes],
|
491
|
-
version: str
|
492
|
-
) -> TypedOrderedDict[str, Any]:
|
486
|
+
kek: KeyEncryptionKey, cek: bytes, iv: Optional[bytes], version: str
|
487
|
+
) -> TypedOrderedDict[str, Any]:
|
493
488
|
"""
|
494
489
|
Generates and returns the encryption metadata as a dict.
|
495
490
|
|
@@ -506,7 +501,7 @@ def _generate_encryption_data_dict(
|
|
506
501
|
# For V2, we include the encryption version in the wrapped key.
|
507
502
|
elif version == _ENCRYPTION_PROTOCOL_V2:
|
508
503
|
# We must pad the version to 8 bytes for AES Keywrap algorithms
|
509
|
-
to_wrap = _ENCRYPTION_PROTOCOL_V2.encode().ljust(8, b
|
504
|
+
to_wrap = _ENCRYPTION_PROTOCOL_V2.encode().ljust(8, b"\0") + cek
|
510
505
|
wrapped_cek = kek.wrap_key(to_wrap)
|
511
506
|
else:
|
512
507
|
raise ValueError("Invalid encryption version specified.")
|
@@ -514,31 +509,31 @@ def _generate_encryption_data_dict(
|
|
514
509
|
# Build the encryption_data dict.
|
515
510
|
# Use OrderedDict to comply with Java's ordering requirement.
|
516
511
|
wrapped_content_key = OrderedDict()
|
517
|
-
wrapped_content_key[
|
518
|
-
wrapped_content_key[
|
519
|
-
wrapped_content_key[
|
512
|
+
wrapped_content_key["KeyId"] = kek.get_kid()
|
513
|
+
wrapped_content_key["EncryptedKey"] = encode_base64(wrapped_cek)
|
514
|
+
wrapped_content_key["Algorithm"] = kek.get_key_wrap_algorithm()
|
520
515
|
|
521
516
|
encryption_agent = OrderedDict()
|
522
|
-
encryption_agent[
|
517
|
+
encryption_agent["Protocol"] = version
|
523
518
|
|
524
519
|
if version == _ENCRYPTION_PROTOCOL_V1:
|
525
|
-
encryption_agent[
|
520
|
+
encryption_agent["EncryptionAlgorithm"] = _EncryptionAlgorithm.AES_CBC_256
|
526
521
|
|
527
522
|
elif version == _ENCRYPTION_PROTOCOL_V2:
|
528
|
-
encryption_agent[
|
523
|
+
encryption_agent["EncryptionAlgorithm"] = _EncryptionAlgorithm.AES_GCM_256
|
529
524
|
|
530
525
|
encrypted_region_info = OrderedDict()
|
531
|
-
encrypted_region_info[
|
532
|
-
encrypted_region_info[
|
526
|
+
encrypted_region_info["DataLength"] = _GCM_REGION_DATA_LENGTH
|
527
|
+
encrypted_region_info["NonceLength"] = _GCM_NONCE_LENGTH
|
533
528
|
|
534
529
|
encryption_data_dict: TypedOrderedDict[str, Any] = OrderedDict()
|
535
|
-
encryption_data_dict[
|
536
|
-
encryption_data_dict[
|
530
|
+
encryption_data_dict["WrappedContentKey"] = wrapped_content_key
|
531
|
+
encryption_data_dict["EncryptionAgent"] = encryption_agent
|
537
532
|
if version == _ENCRYPTION_PROTOCOL_V1:
|
538
|
-
encryption_data_dict[
|
533
|
+
encryption_data_dict["ContentEncryptionIV"] = encode_base64(iv)
|
539
534
|
elif version == _ENCRYPTION_PROTOCOL_V2:
|
540
|
-
encryption_data_dict[
|
541
|
-
encryption_data_dict[
|
535
|
+
encryption_data_dict["EncryptedRegionInfo"] = encrypted_region_info
|
536
|
+
encryption_data_dict["KeyWrappingMetadata"] = OrderedDict({"EncryptionLibrary": "Python " + VERSION})
|
542
537
|
|
543
538
|
return encryption_data_dict
|
544
539
|
|
@@ -554,43 +549,42 @@ def _dict_to_encryption_data(encryption_data_dict: Dict[str, Any]) -> _Encryptio
|
|
554
549
|
:rtype: _EncryptionData
|
555
550
|
"""
|
556
551
|
try:
|
557
|
-
protocol = encryption_data_dict[
|
552
|
+
protocol = encryption_data_dict["EncryptionAgent"]["Protocol"]
|
558
553
|
if protocol not in _VALID_ENCRYPTION_PROTOCOLS:
|
559
554
|
raise ValueError("Unsupported encryption version.")
|
560
555
|
except KeyError as exc:
|
561
556
|
raise ValueError("Unsupported encryption version.") from exc
|
562
|
-
wrapped_content_key = encryption_data_dict[
|
563
|
-
wrapped_content_key = _WrappedContentKey(
|
564
|
-
|
565
|
-
|
566
|
-
|
567
|
-
|
568
|
-
|
569
|
-
|
570
|
-
|
571
|
-
|
572
|
-
|
557
|
+
wrapped_content_key = encryption_data_dict["WrappedContentKey"]
|
558
|
+
wrapped_content_key = _WrappedContentKey(
|
559
|
+
wrapped_content_key["Algorithm"],
|
560
|
+
decode_base64_to_bytes(wrapped_content_key["EncryptedKey"]),
|
561
|
+
wrapped_content_key["KeyId"],
|
562
|
+
)
|
563
|
+
|
564
|
+
encryption_agent = encryption_data_dict["EncryptionAgent"]
|
565
|
+
encryption_agent = _EncryptionAgent(encryption_agent["EncryptionAlgorithm"], encryption_agent["Protocol"])
|
566
|
+
|
567
|
+
if "KeyWrappingMetadata" in encryption_data_dict:
|
568
|
+
key_wrapping_metadata = encryption_data_dict["KeyWrappingMetadata"]
|
573
569
|
else:
|
574
570
|
key_wrapping_metadata = None
|
575
571
|
|
576
572
|
# AES-CBC only
|
577
573
|
encryption_iv = None
|
578
|
-
if
|
579
|
-
encryption_iv = decode_base64_to_bytes(encryption_data_dict[
|
574
|
+
if "ContentEncryptionIV" in encryption_data_dict:
|
575
|
+
encryption_iv = decode_base64_to_bytes(encryption_data_dict["ContentEncryptionIV"])
|
580
576
|
|
581
577
|
# AES-GCM only
|
582
578
|
region_info = None
|
583
|
-
if
|
584
|
-
encrypted_region_info = encryption_data_dict[
|
585
|
-
region_info = _EncryptedRegionInfo(
|
586
|
-
|
587
|
-
|
588
|
-
|
589
|
-
encryption_data = _EncryptionData(
|
590
|
-
|
591
|
-
|
592
|
-
wrapped_content_key,
|
593
|
-
key_wrapping_metadata)
|
579
|
+
if "EncryptedRegionInfo" in encryption_data_dict:
|
580
|
+
encrypted_region_info = encryption_data_dict["EncryptedRegionInfo"]
|
581
|
+
region_info = _EncryptedRegionInfo(
|
582
|
+
encrypted_region_info["DataLength"], encrypted_region_info["NonceLength"], _GCM_TAG_LENGTH
|
583
|
+
)
|
584
|
+
|
585
|
+
encryption_data = _EncryptionData(
|
586
|
+
encryption_iv, region_info, encryption_agent, wrapped_content_key, key_wrapping_metadata
|
587
|
+
)
|
594
588
|
|
595
589
|
return encryption_data
|
596
590
|
|
@@ -614,7 +608,7 @@ def _generate_AES_CBC_cipher(cek: bytes, iv: bytes) -> Cipher:
|
|
614
608
|
def _validate_and_unwrap_cek(
|
615
609
|
encryption_data: _EncryptionData,
|
616
610
|
key_encryption_key: Optional[KeyEncryptionKey] = None,
|
617
|
-
key_resolver: Optional[Callable[[str], KeyEncryptionKey]] = None
|
611
|
+
key_resolver: Optional[Callable[[str], KeyEncryptionKey]] = None,
|
618
612
|
) -> bytes:
|
619
613
|
"""
|
620
614
|
Extracts and returns the content_encryption_key stored in the encryption_data object
|
@@ -636,15 +630,15 @@ def _validate_and_unwrap_cek(
|
|
636
630
|
:rtype: bytes
|
637
631
|
"""
|
638
632
|
|
639
|
-
_validate_not_none(
|
633
|
+
_validate_not_none("encrypted_key", encryption_data.wrapped_content_key.encrypted_key)
|
640
634
|
|
641
635
|
# Validate we have the right info for the specified version
|
642
636
|
if encryption_data.encryption_agent.protocol == _ENCRYPTION_PROTOCOL_V1:
|
643
|
-
_validate_not_none(
|
637
|
+
_validate_not_none("content_encryption_IV", encryption_data.content_encryption_IV)
|
644
638
|
elif encryption_data.encryption_agent.protocol in _ENCRYPTION_V2_PROTOCOLS:
|
645
|
-
_validate_not_none(
|
639
|
+
_validate_not_none("encrypted_region_info", encryption_data.encrypted_region_info)
|
646
640
|
else:
|
647
|
-
raise ValueError(
|
641
|
+
raise ValueError("Specified encryption version is not supported.")
|
648
642
|
|
649
643
|
content_encryption_key: Optional[bytes] = None
|
650
644
|
|
@@ -654,29 +648,29 @@ def _validate_and_unwrap_cek(
|
|
654
648
|
|
655
649
|
if key_encryption_key is None:
|
656
650
|
raise ValueError("Unable to decrypt. key_resolver and key_encryption_key cannot both be None.")
|
657
|
-
if not hasattr(key_encryption_key,
|
658
|
-
raise AttributeError(_ERROR_OBJECT_INVALID.format(
|
659
|
-
if not hasattr(key_encryption_key,
|
660
|
-
raise AttributeError(_ERROR_OBJECT_INVALID.format(
|
651
|
+
if not hasattr(key_encryption_key, "get_kid") or not callable(key_encryption_key.get_kid):
|
652
|
+
raise AttributeError(_ERROR_OBJECT_INVALID.format("key encryption key", "get_kid"))
|
653
|
+
if not hasattr(key_encryption_key, "unwrap_key") or not callable(key_encryption_key.unwrap_key):
|
654
|
+
raise AttributeError(_ERROR_OBJECT_INVALID.format("key encryption key", "unwrap_key"))
|
661
655
|
if encryption_data.wrapped_content_key.key_id != key_encryption_key.get_kid():
|
662
|
-
raise ValueError(
|
656
|
+
raise ValueError("Provided or resolved key-encryption-key does not match the id of key used to encrypt.")
|
663
657
|
# Will throw an exception if the specified algorithm is not supported.
|
664
658
|
content_encryption_key = key_encryption_key.unwrap_key(
|
665
|
-
encryption_data.wrapped_content_key.encrypted_key,
|
666
|
-
|
659
|
+
encryption_data.wrapped_content_key.encrypted_key, encryption_data.wrapped_content_key.algorithm
|
660
|
+
)
|
667
661
|
|
668
662
|
# For V2, the version is included with the cek. We need to validate it
|
669
663
|
# and remove it from the actual cek.
|
670
664
|
if encryption_data.encryption_agent.protocol in _ENCRYPTION_V2_PROTOCOLS:
|
671
|
-
version_2_bytes = encryption_data.encryption_agent.protocol.encode().ljust(8, b
|
672
|
-
cek_version_bytes = content_encryption_key[:len(version_2_bytes)]
|
665
|
+
version_2_bytes = encryption_data.encryption_agent.protocol.encode().ljust(8, b"\0")
|
666
|
+
cek_version_bytes = content_encryption_key[: len(version_2_bytes)]
|
673
667
|
if cek_version_bytes != version_2_bytes:
|
674
|
-
raise ValueError(
|
668
|
+
raise ValueError("The encryption metadata is not valid and may have been modified.")
|
675
669
|
|
676
670
|
# Remove version from the start of the cek.
|
677
|
-
content_encryption_key = content_encryption_key[len(version_2_bytes):]
|
671
|
+
content_encryption_key = content_encryption_key[len(version_2_bytes) :]
|
678
672
|
|
679
|
-
_validate_not_none(
|
673
|
+
_validate_not_none("content_encryption_key", content_encryption_key)
|
680
674
|
|
681
675
|
return content_encryption_key
|
682
676
|
|
@@ -685,7 +679,7 @@ def _decrypt_message(
|
|
685
679
|
message: bytes,
|
686
680
|
encryption_data: _EncryptionData,
|
687
681
|
key_encryption_key: Optional[KeyEncryptionKey] = None,
|
688
|
-
resolver: Optional[Callable[[str], KeyEncryptionKey]] = None
|
682
|
+
resolver: Optional[Callable[[str], KeyEncryptionKey]] = None,
|
689
683
|
) -> bytes:
|
690
684
|
"""
|
691
685
|
Decrypts the given ciphertext using AES256 in CBC mode with 128 bit padding.
|
@@ -710,7 +704,7 @@ def _decrypt_message(
|
|
710
704
|
:return: The decrypted plaintext.
|
711
705
|
:rtype: bytes
|
712
706
|
"""
|
713
|
-
_validate_not_none(
|
707
|
+
_validate_not_none("message", message)
|
714
708
|
content_encryption_key = _validate_and_unwrap_cek(encryption_data, key_encryption_key, resolver)
|
715
709
|
|
716
710
|
if encryption_data.encryption_agent.protocol == _ENCRYPTION_PROTOCOL_V1:
|
@@ -721,11 +715,11 @@ def _decrypt_message(
|
|
721
715
|
|
722
716
|
# decrypt data
|
723
717
|
decryptor = cipher.decryptor()
|
724
|
-
decrypted_data =
|
718
|
+
decrypted_data = decryptor.update(message) + decryptor.finalize()
|
725
719
|
|
726
720
|
# unpad data
|
727
721
|
unpadder = PKCS7(128).unpadder()
|
728
|
-
decrypted_data =
|
722
|
+
decrypted_data = unpadder.update(decrypted_data) + unpadder.finalize()
|
729
723
|
|
730
724
|
elif encryption_data.encryption_agent.protocol in _ENCRYPTION_V2_PROTOCOLS:
|
731
725
|
block_info = encryption_data.encrypted_region_info
|
@@ -745,7 +739,7 @@ def _decrypt_message(
|
|
745
739
|
decrypted_data = aesgcm.decrypt(nonce, ciphertext_with_tag, None)
|
746
740
|
|
747
741
|
else:
|
748
|
-
raise ValueError(
|
742
|
+
raise ValueError("Specified encryption version is not supported.")
|
749
743
|
|
750
744
|
return decrypted_data
|
751
745
|
|
@@ -773,8 +767,8 @@ def encrypt_blob(blob: bytes, key_encryption_key: KeyEncryptionKey, version: str
|
|
773
767
|
:rtype: (str, bytes)
|
774
768
|
"""
|
775
769
|
|
776
|
-
_validate_not_none(
|
777
|
-
_validate_not_none(
|
770
|
+
_validate_not_none("blob", blob)
|
771
|
+
_validate_not_none("key_encryption_key", key_encryption_key)
|
778
772
|
_validate_key_encryption_key_wrap(key_encryption_key)
|
779
773
|
|
780
774
|
if version == _ENCRYPTION_PROTOCOL_V1:
|
@@ -805,16 +799,16 @@ def encrypt_blob(blob: bytes, key_encryption_key: KeyEncryptionKey, version: str
|
|
805
799
|
else:
|
806
800
|
raise ValueError("Invalid encryption version specified.")
|
807
801
|
|
808
|
-
encryption_data = _generate_encryption_data_dict(
|
809
|
-
|
810
|
-
|
802
|
+
encryption_data = _generate_encryption_data_dict(
|
803
|
+
key_encryption_key, content_encryption_key, initialization_vector, version
|
804
|
+
)
|
805
|
+
encryption_data["EncryptionMode"] = "FullBlob"
|
811
806
|
|
812
807
|
return dumps(encryption_data), encrypted_data
|
813
808
|
|
814
809
|
|
815
810
|
def generate_blob_encryption_data(
|
816
|
-
key_encryption_key: Optional[KeyEncryptionKey],
|
817
|
-
version: str
|
811
|
+
key_encryption_key: Optional[KeyEncryptionKey], version: str
|
818
812
|
) -> Tuple[Optional[bytes], Optional[bytes], Optional[str]]:
|
819
813
|
"""
|
820
814
|
Generates the encryption_metadata for the blob.
|
@@ -836,24 +830,23 @@ def generate_blob_encryption_data(
|
|
836
830
|
# Initialization vector only needed for V1
|
837
831
|
if version == _ENCRYPTION_PROTOCOL_V1:
|
838
832
|
initialization_vector = os.urandom(16)
|
839
|
-
encryption_data_dict = _generate_encryption_data_dict(
|
840
|
-
|
841
|
-
|
842
|
-
|
843
|
-
encryption_data_dict['EncryptionMode'] = 'FullBlob'
|
833
|
+
encryption_data_dict = _generate_encryption_data_dict(
|
834
|
+
key_encryption_key, content_encryption_key, initialization_vector, version
|
835
|
+
)
|
836
|
+
encryption_data_dict["EncryptionMode"] = "FullBlob"
|
844
837
|
encryption_data = dumps(encryption_data_dict)
|
845
838
|
|
846
839
|
return content_encryption_key, initialization_vector, encryption_data
|
847
840
|
|
848
841
|
|
849
842
|
def decrypt_blob( # pylint: disable=too-many-locals,too-many-statements
|
850
|
-
|
851
|
-
|
852
|
-
|
853
|
-
|
854
|
-
|
855
|
-
|
856
|
-
|
843
|
+
require_encryption: bool,
|
844
|
+
key_encryption_key: Optional[KeyEncryptionKey],
|
845
|
+
key_resolver: Optional[Callable[[str], KeyEncryptionKey]],
|
846
|
+
content: bytes,
|
847
|
+
start_offset: int,
|
848
|
+
end_offset: int,
|
849
|
+
response_headers: Dict[str, Any],
|
857
850
|
) -> bytes:
|
858
851
|
"""
|
859
852
|
Decrypts the given blob contents and returns only the requested range.
|
@@ -885,39 +878,40 @@ def decrypt_blob( # pylint: disable=too-many-locals,too-many-statements
|
|
885
878
|
:rtype: bytes
|
886
879
|
"""
|
887
880
|
try:
|
888
|
-
encryption_data = _dict_to_encryption_data(loads(response_headers[
|
881
|
+
encryption_data = _dict_to_encryption_data(loads(response_headers["x-ms-meta-encryptiondata"]))
|
889
882
|
except Exception as exc: # pylint: disable=broad-except
|
890
883
|
if require_encryption:
|
891
884
|
raise ValueError(
|
892
|
-
|
893
|
-
|
885
|
+
"Encryption required, but received data does not contain appropriate metadata."
|
886
|
+
+ "Data was either not encrypted or metadata has been lost."
|
887
|
+
) from exc
|
894
888
|
|
895
889
|
return content
|
896
890
|
|
897
891
|
algorithm = encryption_data.encryption_agent.encryption_algorithm
|
898
|
-
if algorithm not in(_EncryptionAlgorithm.AES_CBC_256, _EncryptionAlgorithm.AES_GCM_256):
|
899
|
-
raise ValueError(
|
892
|
+
if algorithm not in (_EncryptionAlgorithm.AES_CBC_256, _EncryptionAlgorithm.AES_GCM_256):
|
893
|
+
raise ValueError("Specified encryption algorithm is not supported.")
|
900
894
|
|
901
895
|
version = encryption_data.encryption_agent.protocol
|
902
896
|
if version not in _VALID_ENCRYPTION_PROTOCOLS:
|
903
|
-
raise ValueError(
|
897
|
+
raise ValueError("Specified encryption version is not supported.")
|
904
898
|
|
905
899
|
content_encryption_key = _validate_and_unwrap_cek(encryption_data, key_encryption_key, key_resolver)
|
906
900
|
|
907
901
|
if version == _ENCRYPTION_PROTOCOL_V1:
|
908
|
-
blob_type = response_headers[
|
902
|
+
blob_type = response_headers["x-ms-blob-type"]
|
909
903
|
|
910
904
|
iv: Optional[bytes] = None
|
911
905
|
unpad = False
|
912
|
-
if
|
913
|
-
content_range = response_headers[
|
906
|
+
if "content-range" in response_headers:
|
907
|
+
content_range = response_headers["content-range"]
|
914
908
|
# Format: 'bytes x-y/size'
|
915
909
|
|
916
910
|
# Ignore the word 'bytes'
|
917
|
-
content_range = content_range.split(
|
911
|
+
content_range = content_range.split(" ")
|
918
912
|
|
919
|
-
content_range = content_range[1].split(
|
920
|
-
content_range = content_range[1].split(
|
913
|
+
content_range = content_range[1].split("-")
|
914
|
+
content_range = content_range[1].split("/")
|
921
915
|
end_range = int(content_range[0])
|
922
916
|
blob_size = int(content_range[1])
|
923
917
|
|
@@ -934,7 +928,7 @@ def decrypt_blob( # pylint: disable=too-many-locals,too-many-statements
|
|
934
928
|
unpad = True
|
935
929
|
iv = encryption_data.content_encryption_IV
|
936
930
|
|
937
|
-
if blob_type ==
|
931
|
+
if blob_type == "PageBlob":
|
938
932
|
unpad = False
|
939
933
|
|
940
934
|
if iv is None:
|
@@ -948,7 +942,7 @@ def decrypt_blob( # pylint: disable=too-many-locals,too-many-statements
|
|
948
942
|
unpadder = PKCS7(128).unpadder()
|
949
943
|
content = unpadder.update(content) + unpadder.finalize()
|
950
944
|
|
951
|
-
return content[start_offset: len(content) - end_offset]
|
945
|
+
return content[start_offset : len(content) - end_offset]
|
952
946
|
|
953
947
|
if version in _ENCRYPTION_V2_PROTOCOLS:
|
954
948
|
# We assume the content contains only full encryption regions
|
@@ -967,7 +961,7 @@ def decrypt_blob( # pylint: disable=too-many-locals,too-many-statements
|
|
967
961
|
while offset < total_size:
|
968
962
|
# Process one encryption region at a time
|
969
963
|
process_size = min(region_length, total_size)
|
970
|
-
encrypted_region = content[offset:offset + process_size]
|
964
|
+
encrypted_region = content[offset : offset + process_size]
|
971
965
|
|
972
966
|
# First bytes are the nonce
|
973
967
|
nonce = encrypted_region[:nonce_length]
|
@@ -982,13 +976,11 @@ def decrypt_blob( # pylint: disable=too-many-locals,too-many-statements
|
|
982
976
|
# Read the caller requested data from the decrypted content
|
983
977
|
return decrypted_content[start_offset:end_offset]
|
984
978
|
|
985
|
-
raise ValueError(
|
979
|
+
raise ValueError("Specified encryption version is not supported.")
|
986
980
|
|
987
981
|
|
988
982
|
def get_blob_encryptor_and_padder(
|
989
|
-
cek: Optional[bytes],
|
990
|
-
iv: Optional[bytes],
|
991
|
-
should_pad: bool
|
983
|
+
cek: Optional[bytes], iv: Optional[bytes], should_pad: bool
|
992
984
|
) -> Tuple[Optional["AEADEncryptionContext"], Optional["PaddingContext"]]:
|
993
985
|
encryptor = None
|
994
986
|
padder = None
|
@@ -1022,13 +1014,13 @@ def encrypt_queue_message(message: str, key_encryption_key: KeyEncryptionKey, ve
|
|
1022
1014
|
:rtype: str
|
1023
1015
|
"""
|
1024
1016
|
|
1025
|
-
_validate_not_none(
|
1026
|
-
_validate_not_none(
|
1017
|
+
_validate_not_none("message", message)
|
1018
|
+
_validate_not_none("key_encryption_key", key_encryption_key)
|
1027
1019
|
_validate_key_encryption_key_wrap(key_encryption_key)
|
1028
1020
|
|
1029
1021
|
# Queue encoding functions all return unicode strings, and encryption should
|
1030
1022
|
# operate on binary strings.
|
1031
|
-
message_as_bytes: bytes = message.encode(
|
1023
|
+
message_as_bytes: bytes = message.encode("utf-8")
|
1032
1024
|
|
1033
1025
|
if version == _ENCRYPTION_PROTOCOL_V1:
|
1034
1026
|
# AES256 CBC uses 256 bit (32 byte) keys and always with 16 byte blocks
|
@@ -1062,11 +1054,12 @@ def encrypt_queue_message(message: str, key_encryption_key: KeyEncryptionKey, ve
|
|
1062
1054
|
raise ValueError("Invalid encryption version specified.")
|
1063
1055
|
|
1064
1056
|
# Build the dictionary structure.
|
1065
|
-
queue_message = {
|
1066
|
-
|
1067
|
-
|
1068
|
-
|
1069
|
-
|
1057
|
+
queue_message = {
|
1058
|
+
"EncryptedMessageContents": encode_base64(encrypted_data),
|
1059
|
+
"EncryptionData": _generate_encryption_data_dict(
|
1060
|
+
key_encryption_key, content_encryption_key, initialization_vector, version
|
1061
|
+
),
|
1062
|
+
}
|
1070
1063
|
|
1071
1064
|
return dumps(queue_message)
|
1072
1065
|
|
@@ -1076,7 +1069,7 @@ def decrypt_queue_message(
|
|
1076
1069
|
response: "PipelineResponse",
|
1077
1070
|
require_encryption: bool,
|
1078
1071
|
key_encryption_key: Optional[KeyEncryptionKey],
|
1079
|
-
resolver: Optional[Callable[[str], KeyEncryptionKey]]
|
1072
|
+
resolver: Optional[Callable[[str], KeyEncryptionKey]],
|
1080
1073
|
) -> str:
|
1081
1074
|
"""
|
1082
1075
|
Returns the decrypted message contents from an EncryptedQueueMessage.
|
@@ -1106,22 +1099,22 @@ def decrypt_queue_message(
|
|
1106
1099
|
try:
|
1107
1100
|
deserialized_message: Dict[str, Any] = loads(message)
|
1108
1101
|
|
1109
|
-
encryption_data = _dict_to_encryption_data(deserialized_message[
|
1110
|
-
decoded_data = decode_base64_to_bytes(deserialized_message[
|
1102
|
+
encryption_data = _dict_to_encryption_data(deserialized_message["EncryptionData"])
|
1103
|
+
decoded_data = decode_base64_to_bytes(deserialized_message["EncryptedMessageContents"])
|
1111
1104
|
except (KeyError, ValueError) as exc:
|
1112
1105
|
# Message was not json formatted and so was not encrypted
|
1113
1106
|
# or the user provided a json formatted message
|
1114
1107
|
# or the metadata was malformed.
|
1115
1108
|
if require_encryption:
|
1116
1109
|
raise ValueError(
|
1117
|
-
|
1118
|
-
|
1110
|
+
"Encryption required, but received message does not contain appropriate metatadata. "
|
1111
|
+
+ "Message was either not encrypted or metadata was incorrect."
|
1112
|
+
) from exc
|
1119
1113
|
|
1120
1114
|
return message
|
1121
1115
|
try:
|
1122
|
-
return _decrypt_message(decoded_data, encryption_data, key_encryption_key, resolver).decode(
|
1116
|
+
return _decrypt_message(decoded_data, encryption_data, key_encryption_key, resolver).decode("utf-8")
|
1123
1117
|
except Exception as error:
|
1124
1118
|
raise HttpResponseError(
|
1125
|
-
message="Decryption failed.",
|
1126
|
-
|
1127
|
-
error=error) from error
|
1119
|
+
message="Decryption failed.", response=response, error=error # type: ignore [arg-type]
|
1120
|
+
) from error
|