azure-storage-blob 12.22.0__py3-none-any.whl → 12.23.0__py3-none-any.whl

azure/storage/blob/_generated/operations/_service_operations.py

@@ -15,17 +15,17 @@ from azure.core.exceptions import (
     ResourceExistsError,
     ResourceNotFoundError,
     ResourceNotModifiedError,
+    StreamClosedError,
+    StreamConsumedError,
     map_error,
 )
 from azure.core.pipeline import PipelineResponse
-from azure.core.pipeline.transport import HttpResponse
-from azure.core.rest import HttpRequest
+from azure.core.rest import HttpRequest, HttpResponse
 from azure.core.tracing.decorator import distributed_trace
 from azure.core.utils import case_insensitive_dict
 
 from .. import models as _models
 from .._serialization import Serializer
-from .._vendor import _convert_request
 
 if sys.version_info >= (3, 9):
     from collections.abc import MutableMapping
@@ -427,7 +427,6 @@ class ServiceOperations:
             headers=_headers,
             params=_params,
         )
-        _request = _convert_request(_request)
         _request.url = self._client.format_url(_request.url)
 
         _stream = False
@@ -497,7 +496,6 @@ class ServiceOperations:
             headers=_headers,
             params=_params,
         )
-        _request = _convert_request(_request)
         _request.url = self._client.format_url(_request.url)
 
         _stream = False
@@ -519,7 +517,7 @@ class ServiceOperations:
         response_headers["x-ms-request-id"] = self._deserialize("str", response.headers.get("x-ms-request-id"))
         response_headers["x-ms-version"] = self._deserialize("str", response.headers.get("x-ms-version"))
 
-        deserialized = self._deserialize("StorageServiceProperties", pipeline_response)
+        deserialized = self._deserialize("StorageServiceProperties", pipeline_response.http_response)
 
         if cls:
             return cls(pipeline_response, deserialized, response_headers)  # type: ignore
@@ -572,7 +570,6 @@ class ServiceOperations:
             headers=_headers,
             params=_params,
         )
-        _request = _convert_request(_request)
         _request.url = self._client.format_url(_request.url)
 
         _stream = False
@@ -595,7 +592,7 @@ class ServiceOperations:
         response_headers["x-ms-version"] = self._deserialize("str", response.headers.get("x-ms-version"))
         response_headers["Date"] = self._deserialize("rfc-1123", response.headers.get("Date"))
 
-        deserialized = self._deserialize("StorageServiceStats", pipeline_response)
+        deserialized = self._deserialize("StorageServiceStats", pipeline_response.http_response)
 
         if cls:
             return cls(pipeline_response, deserialized, response_headers)  # type: ignore
@@ -676,7 +673,6 @@ class ServiceOperations:
             headers=_headers,
             params=_params,
         )
-        _request = _convert_request(_request)
         _request.url = self._client.format_url(_request.url)
 
         _stream = False
@@ -698,7 +694,7 @@ class ServiceOperations:
         response_headers["x-ms-request-id"] = self._deserialize("str", response.headers.get("x-ms-request-id"))
         response_headers["x-ms-version"] = self._deserialize("str", response.headers.get("x-ms-version"))
 
-        deserialized = self._deserialize("ListContainersSegmentResponse", pipeline_response)
+        deserialized = self._deserialize("ListContainersSegmentResponse", pipeline_response.http_response)
 
         if cls:
             return cls(pipeline_response, deserialized, response_headers)  # type: ignore
@@ -761,7 +757,6 @@ class ServiceOperations:
             headers=_headers,
             params=_params,
         )
-        _request = _convert_request(_request)
         _request.url = self._client.format_url(_request.url)
 
         _stream = False
@@ -784,7 +779,7 @@ class ServiceOperations:
         response_headers["x-ms-version"] = self._deserialize("str", response.headers.get("x-ms-version"))
         response_headers["Date"] = self._deserialize("rfc-1123", response.headers.get("Date"))
 
-        deserialized = self._deserialize("UserDelegationKey", pipeline_response)
+        deserialized = self._deserialize("UserDelegationKey", pipeline_response.http_response)
 
         if cls:
             return cls(pipeline_response, deserialized, response_headers)  # type: ignore
@@ -835,7 +830,6 @@ class ServiceOperations:
             headers=_headers,
             params=_params,
         )
-        _request = _convert_request(_request)
         _request.url = self._client.format_url(_request.url)
 
         _stream = False
@@ -923,9 +917,9 @@ class ServiceOperations:
             headers=_headers,
             params=_params,
         )
-        _request = _convert_request(_request)
         _request.url = self._client.format_url(_request.url)
 
+        _decompress = kwargs.pop("decompress", True)
         _stream = True
         pipeline_response: PipelineResponse = self._client._pipeline.run(  # pylint: disable=protected-access
             _request, stream=_stream, **kwargs
@@ -934,6 +928,10 @@ class ServiceOperations:
         response = pipeline_response.http_response
 
         if response.status_code not in [200]:
+            try:
+                response.read()  # Load the body in memory and close the socket
+            except (StreamConsumedError, StreamClosedError):
+                pass
             map_error(status_code=response.status_code, response=response, error_map=error_map)
             error = self._deserialize.failsafe_deserialize(_models.StorageError, pipeline_response)
             raise HttpResponseError(response=response, model=error)
@@ -943,7 +941,7 @@ class ServiceOperations:
         response_headers["x-ms-request-id"] = self._deserialize("str", response.headers.get("x-ms-request-id"))
         response_headers["x-ms-version"] = self._deserialize("str", response.headers.get("x-ms-version"))
 
-        deserialized = response.stream_download(self._client._pipeline)
+        deserialized = response.stream_download(self._client._pipeline, decompress=_decompress)
 
         if cls:
             return cls(pipeline_response, deserialized, response_headers)  # type: ignore
@@ -1025,7 +1023,6 @@ class ServiceOperations:
             headers=_headers,
             params=_params,
         )
-        _request = _convert_request(_request)
         _request.url = self._client.format_url(_request.url)
 
         _stream = False
@@ -1048,7 +1045,7 @@ class ServiceOperations:
         response_headers["x-ms-version"] = self._deserialize("str", response.headers.get("x-ms-version"))
         response_headers["Date"] = self._deserialize("rfc-1123", response.headers.get("Date"))
 
-        deserialized = self._deserialize("FilterBlobSegment", pipeline_response)
+        deserialized = self._deserialize("FilterBlobSegment", pipeline_response.http_response)
 
         if cls:
             return cls(pipeline_response, deserialized, response_headers)  # type: ignore

azure/storage/blob/_serialize.py

@@ -56,6 +56,7 @@ _SUPPORTED_API_VERSIONS = [
     '2023-11-03',
     '2024-05-04',
     '2024-08-04',
+    '2024-11-04',
 ]
 
 

azure/storage/blob/_shared/base_client.py

@@ -76,6 +76,7 @@ class StorageAccountHostsMixin(object):  # pylint: disable=too-many-instance-att
         self._location_mode = kwargs.get("_location_mode", LocationMode.PRIMARY)
         self._hosts = kwargs.get("_hosts")
         self.scheme = parsed_url.scheme
+        self._is_localhost = False
 
         if service not in ["blob", "queue", "file-share", "dfs"]:
             raise ValueError(f"Invalid service: {service}")
@@ -85,6 +86,7 @@ class StorageAccountHostsMixin(object):  # pylint: disable=too-many-instance-att
         self.account_name = account[0] if len(account) > 1 else None
         if not self.account_name and parsed_url.netloc.startswith("localhost") \
                 or parsed_url.netloc.startswith("127.0.0.1"):
+            self._is_localhost = True
             self.account_name = parsed_url.path.strip("/")
 
         self.credential = _format_shared_key_credential(self.account_name, credential)

azure/storage/blob/_shared/policies.py

@@ -35,11 +35,6 @@ from .authentication import AzureSigningError, StorageHttpChallenge
 from .constants import DEFAULT_OAUTH_SCOPE
 from .models import LocationMode
 
-try:
-    _unicode_type = unicode # type: ignore
-except NameError:
-    _unicode_type = str
-
 if TYPE_CHECKING:
     from azure.core.credentials import TokenCredential
     from azure.core.pipeline.transport import (  # pylint: disable=non-abstract-transport-import
@@ -52,7 +47,7 @@ _LOGGER = logging.getLogger(__name__)
 
 
 def encode_base64(data):
-    if isinstance(data, _unicode_type):
+    if isinstance(data, str):
         data = data.encode('utf-8')
     encoded = base64.b64encode(data)
     return encoded.decode('utf-8')
@@ -95,10 +90,14 @@ def is_retry(response, mode):   # pylint: disable=too-many-return-statements
         if status in [501, 505]:
             return False
         return True
+    return False
+
+
+def is_checksum_retry(response):
     # retry if invalid content md5
     if response.context.get('validate_content', False) and response.http_response.headers.get('content-md5'):
         computed_md5 = response.http_request.headers.get('content-md5', None) or \
-                       encode_base64(StorageContentValidation.get_content_md5(response.http_response.body()))
+                            encode_base64(StorageContentValidation.get_content_md5(response.http_response.body()))
         if response.http_response.headers['content-md5'] != computed_md5:
             return True
     return False
@@ -301,7 +300,7 @@ class StorageResponseHook(HTTPPolicy):
 
         response = self.next.send(request)
 
-        will_retry = is_retry(response, request.context.options.get('mode'))
+        will_retry = is_retry(response, request.context.options.get('mode')) or is_checksum_retry(response)
         # Auth error could come from Bearer challenge, in which case this request will be made again
         is_auth_error = response.http_response.status_code == 401
         should_update_counts = not (will_retry or is_auth_error)
@@ -527,7 +526,7 @@ class StorageRetryPolicy(HTTPPolicy):
         while retries_remaining:
             try:
                 response = self.next.send(request)
-                if is_retry(response, retry_settings['mode']):
+                if is_retry(response, retry_settings['mode']) or is_checksum_retry(response):
                     retries_remaining = self.increment(
                         retry_settings,
                         request=request.http_request,

azure/storage/blob/_shared/policies_async.py

@@ -10,12 +10,12 @@ import logging
 import random
 from typing import Any, Dict, TYPE_CHECKING
 
-from azure.core.exceptions import AzureError
+from azure.core.exceptions import AzureError, StreamClosedError, StreamConsumedError
 from azure.core.pipeline.policies import AsyncBearerTokenCredentialPolicy, AsyncHTTPPolicy
 
 from .authentication import AzureSigningError, StorageHttpChallenge
 from .constants import DEFAULT_OAUTH_SCOPE
-from .policies import is_retry, StorageRetryPolicy
+from .policies import encode_base64, is_retry, StorageContentValidation, StorageRetryPolicy
 
 if TYPE_CHECKING:
     from azure.core.credentials_async import AsyncTokenCredential
@@ -42,6 +42,20 @@ async def retry_hook(settings, **kwargs):
                 **kwargs)
 
 
+async def is_checksum_retry(response):
+    # retry if invalid content md5
+    if response.context.get('validate_content', False) and response.http_response.headers.get('content-md5'):
+        try:
+            await response.http_response.read()  # Load the body in memory and close the socket
+        except (StreamClosedError, StreamConsumedError):
+            pass
+        computed_md5 = response.http_request.headers.get('content-md5', None) or \
+                       encode_base64(StorageContentValidation.get_content_md5(response.http_response.content))
+        if response.http_response.headers['content-md5'] != computed_md5:
+            return True
+    return False
+
+
 class AsyncStorageResponseHook(AsyncHTTPPolicy):
 
     def __init__(self, **kwargs):  # pylint: disable=unused-argument
@@ -64,9 +78,8 @@ class AsyncStorageResponseHook(AsyncHTTPPolicy):
             request.context.options.pop('raw_response_hook', self._response_callback)
 
         response = await self.next.send(request)
-        await response.http_response.load_body()
+        will_retry = is_retry(response, request.context.options.get('mode')) or await is_checksum_retry(response)
 
-        will_retry = is_retry(response, request.context.options.get('mode'))
         # Auth error could come from Bearer challenge, in which case this request will be made again
         is_auth_error = response.http_response.status_code == 401
         should_update_counts = not (will_retry or is_auth_error)
@@ -112,7 +125,7 @@ class AsyncStorageRetryPolicy(StorageRetryPolicy):
         while retries_remaining:
             try:
                 response = await self.next.send(request)
-                if is_retry(response, retry_settings['mode']):
+                if is_retry(response, retry_settings['mode']) or await is_checksum_retry(response):
                     retries_remaining = self.increment(
                         retry_settings,
                         request=request.http_request,

azure/storage/blob/_shared/shared_access_signature.py

@@ -107,8 +107,17 @@ class SharedAccessSignature(object):
         self.account_key = account_key
         self.x_ms_version = x_ms_version
 
-    def generate_account(self, services, resource_types, permission, expiry, start=None,
-                         ip=None, protocol=None, **kwargs) -> str:
+    def generate_account(
+        self, services,
+        resource_types,
+        permission,
+        expiry,
+        start=None,
+        ip=None,
+        protocol=None,
+        sts_hook=None,
+        **kwargs
+    ) -> str:
         '''
         Generates a shared access signature for the account.
         Use the returned signature with the sas_token parameter of the service
@@ -152,6 +161,10 @@ class SharedAccessSignature(object):
         :keyword str encryption_scope:
             Optional. If specified, this is the encryption scope to use when sending requests
             authorized with this SAS URI.
+        :param sts_hook:
+            For debugging purposes only. If provided, the hook is called with the string to sign
+            that was used to generate the SAS.
+        :type sts_hook: Optional[Callable[[str], None]]
         :returns: The generated SAS token for the account.
         :rtype: str
         '''
@@ -161,12 +174,16 @@ class SharedAccessSignature(object):
         sas.add_encryption_scope(**kwargs)
         sas.add_account_signature(self.account_name, self.account_key)
 
+        if sts_hook is not None:
+            sts_hook(sas.string_to_sign)
+
         return sas.get_token()
 
 
 class _SharedAccessHelper(object):
     def __init__(self):
         self.query_dict = {}
+        self.string_to_sign = ""
 
     def _add_query(self, name, val):
         if val:
@@ -229,6 +246,7 @@ class _SharedAccessHelper(object):
 
         self._add_query(QueryStringConstants.SIGNED_SIGNATURE,
                         sign_string(account_key, string_to_sign))
+        self.string_to_sign = string_to_sign
 
     def get_token(self) -> str:
         return '&'.join([f'{n}={url_quote(v)}' for n, v in self.query_dict.items() if v is not None])

azure/storage/blob/_shared_access_signature.py

@@ -5,7 +5,10 @@
 # --------------------------------------------------------------------------
 # pylint: disable=docstring-keyword-should-match-keyword-only
 
-from typing import Union, Optional, Any, TYPE_CHECKING
+from typing import (
+    Any, Callable, Optional, Union,
+    TYPE_CHECKING
+)
 from urllib.parse import parse_qs
 
 from ._shared import sign_string, url_quote
@@ -64,6 +67,7 @@ class BlobSharedAccessSignature(SharedAccessSignature):
         content_encoding: Optional[str] = None,
         content_language: Optional[str] = None,
         content_type: Optional[str] = None,
+        sts_hook: Optional[Callable[[str], None]] = None,
         **kwargs: Any
     ) -> str:
         '''
@@ -132,6 +136,10 @@ class BlobSharedAccessSignature(SharedAccessSignature):
         :param str content_type:
             Response header value for Content-Type when resource is accessed
             using this shared access signature.
+        :param sts_hook:
+            For debugging purposes only. If provided, the hook is called with the string to sign
+            that was used to generate the SAS.
+        :type sts_hook: Optional[Callable[[str], None]]
         :return: A Shared Access Signature (sas) token.
         :rtype: str
         '''
@@ -155,6 +163,9 @@ class BlobSharedAccessSignature(SharedAccessSignature):
         sas.add_resource_signature(self.account_name, self.account_key, resource_path,
                                    user_delegation_key=self.user_delegation_key)
 
+        if sts_hook is not None:
+            sts_hook(sas.string_to_sign)
+
         return sas.get_token()
 
     def generate_container(
@@ -170,6 +181,7 @@ class BlobSharedAccessSignature(SharedAccessSignature):
         content_encoding: Optional[str] = None,
         content_language: Optional[str] = None,
         content_type: Optional[str] = None,
+        sts_hook: Optional[Callable[[str], None]] = None,
         **kwargs: Any
     ) -> str:
         '''
@@ -228,6 +240,10 @@ class BlobSharedAccessSignature(SharedAccessSignature):
         :param str content_type:
             Response header value for Content-Type when resource is accessed
             using this shared access signature.
+        :param sts_hook:
+            For debugging purposes only. If provided, the hook is called with the string to sign
+            that was used to generate the SAS.
+        :type sts_hook: Optional[Callable[[str], None]]
         :return: A Shared Access Signature (sas) token.
         :rtype: str
         '''
@@ -242,6 +258,10 @@ class BlobSharedAccessSignature(SharedAccessSignature):
         sas.add_info_for_hns_account(**kwargs)
         sas.add_resource_signature(self.account_name, self.account_key, container_name,
                                    user_delegation_key=self.user_delegation_key)
+
+        if sts_hook is not None:
+            sts_hook(sas.string_to_sign)
+
         return sas.get_token()
 
 
@@ -316,6 +336,7 @@ class _BlobSharedAccessHelper(_SharedAccessHelper):
         self._add_query(QueryStringConstants.SIGNED_SIGNATURE,
                         sign_string(account_key if user_delegation_key is None else user_delegation_key.value,
                                     string_to_sign))
+        self.string_to_sign = string_to_sign
 
     def get_token(self) -> str:
         # a conscious decision was made to exclude the timestamp in the generated token
@@ -335,6 +356,7 @@ def generate_account_sas(
     ip: Optional[str] = None,
     *,
     services: Union[Services, str] = Services(blob=True),
+    sts_hook: Optional[Callable[[str], None]] = None,
     **kwargs: Any
 ) -> str:
     """Generates a shared access signature for the blob service.
@@ -376,6 +398,10 @@ def generate_account_sas(
         Specifies the protocol permitted for a request made. The default value is https.
     :keyword str encryption_scope:
         Specifies the encryption scope for a request made so that all write operations will be service encrypted.
+    :keyword sts_hook:
+        For debugging purposes only. If provided, the hook is called with the string to sign
+        that was used to generate the SAS.
+    :paramtype sts_hook: Optional[Callable[[str], None]]
     :return: A Shared Access Signature (sas) token.
     :rtype: str
 
@@ -396,6 +422,7 @@ def generate_account_sas(
         expiry=expiry,
         start=start,
         ip=ip,
+        sts_hook=sts_hook,
         **kwargs
     )
 
@@ -410,6 +437,8 @@ def generate_container_sas(
     start: Optional[Union["datetime", str]] = None,
     policy_id: Optional[str] = None,
     ip: Optional[str] = None,
+    *,
+    sts_hook: Optional[Callable[[str], None]] = None,
     **kwargs: Any
 ) -> str:
     """Generates a shared access signature for a container.
@@ -483,6 +512,10 @@ def generate_container_sas(
     :keyword str correlation_id:
         The correlation id to correlate the storage audit logs with the audit logs used by the principal
         generating and distributing the SAS. This can only be used when generating a SAS with delegation key.
+    :keyword sts_hook:
+        For debugging purposes only. If provided, the hook is called with the string to sign
+        that was used to generate the SAS.
+    :paramtype sts_hook: Optional[Callable[[str], None]]
     :return: A Shared Access Signature (sas) token.
     :rtype: str
 
@@ -515,6 +548,7 @@ def generate_container_sas(
         start=start,
         policy_id=policy_id,
         ip=ip,
+        sts_hook=sts_hook,
         **kwargs
     )
 
@@ -531,6 +565,8 @@ def generate_blob_sas(
     start: Optional[Union["datetime", str]] = None,
     policy_id: Optional[str] = None,
     ip: Optional[str] = None,
+    *,
+    sts_hook: Optional[Callable[[str], None]] = None,
     **kwargs: Any
 ) -> str:
     """Generates a shared access signature for a blob.
@@ -616,6 +652,10 @@ def generate_blob_sas(
     :keyword str correlation_id:
         The correlation id to correlate the storage audit logs with the audit logs used by the principal
         generating and distributing the SAS. This can only be used when generating a SAS with delegation key.
+    :keyword sts_hook:
+        For debugging purposes only. If provided, the hook is called with the string to sign
+        that was used to generate the SAS.
+    :paramtype sts_hook: Optional[Callable[[str], None]]
     :return: A Shared Access Signature (sas) token.
     :rtype: str
     """
@@ -645,6 +685,7 @@ def generate_blob_sas(
         start=start,
         policy_id=policy_id,
         ip=ip,
+        sts_hook=sts_hook,
         **kwargs
     )
 

azure/storage/blob/_version.py

@@ -4,4 +4,4 @@
 # license information.
 # --------------------------------------------------------------------------
 
-VERSION = "12.22.0"
+VERSION = "12.23.0"

azure/storage/blob/aio/_container_client_async.py

@@ -1406,6 +1406,8 @@ class ContainerClient(AsyncStorageAccountHostsMixin, StorageAccountHostsMixin, S
         """
         if len(blobs) == 0:
             return AsyncList([])
+        if self._is_localhost:
+            kwargs['url_prepend'] = self.account_name
 
         reqs, options = _generate_delete_blobs_options(
             self._query_str,
@@ -1485,6 +1487,8 @@ class ContainerClient(AsyncStorageAccountHostsMixin, StorageAccountHostsMixin, S
         :return: An async iterator of responses, one for each blob in order
         :rtype: asynciterator[~azure.core.pipeline.transport.AsyncHttpResponse]
         """
+        if self._is_localhost:
+            kwargs['url_prepend'] = self.account_name
         reqs, options = _generate_set_tiers_options(
             self._query_str,
             self.container_name,
@@ -1544,6 +1548,8 @@ class ContainerClient(AsyncStorageAccountHostsMixin, StorageAccountHostsMixin, S
         :return: An async iterator of responses, one for each blob in order
         :rtype: asynciterator[~azure.core.pipeline.transport.AsyncHttpResponse]
         """
+        if self._is_localhost:
+            kwargs['url_prepend'] = self.account_name
         reqs, options = _generate_set_tiers_options(
             self._query_str,
             self.container_name,